From cb224b3e132bbed82bbe387846b5b62a9d94a905 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Sat, 24 Jun 2023 16:54:18 +0800 Subject: [PATCH 001/337] Add files via upload --- .../neural network/tensorflow/process/LSTM.py | 6 +- .../tensorflow/process/attn_LSTM.py | 4 +- .../neural network/tensorflow/process/cnn.py | 6 +- .../tensorflow/process/cnn_lmix.py | 6 +- .../neural network/tensorflow/process/nn.py | 6 +- .../neural network/tensorflow/process/nn_.py | 6 +- .../tensorflow/process/nn_acc.py | 4 +- .../tensorflow/process/nn_attenuate.py | 6 +- .../tensorflow/process/nn_clipping.py | 6 +- .../tensorflow/process/self_LSTM.py | 6 +- .../tensorflow/process/transformer.py | 55 +++++++++++++++++++ 11 files changed, 83 insertions(+), 28 deletions(-) create mode 100644 Note 7.0 pv documentation/DL/neural network/tensorflow/process/transformer.py diff --git a/Note 7.0 pv documentation/DL/neural network/tensorflow/process/LSTM.py b/Note 7.0 pv documentation/DL/neural network/tensorflow/process/LSTM.py index 03527ea7..3e0647f3 100644 --- a/Note 7.0 pv documentation/DL/neural network/tensorflow/process/LSTM.py +++ b/Note 7.0 pv documentation/DL/neural network/tensorflow/process/LSTM.py @@ -1,14 +1,14 @@ import tensorflow as tf from Note.nn.layer.LSTM import LSTM from Note.nn.layer.dense import dense -import Note.nn.process.optimizer as o +from Note.nn.process.optimizer import Adam # Define a recurrent neural network class with LSTM layers class lstm: def __init__(self): # Initialize the loss function and the optimizer self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.optimizer=o.Adam() + self.optimizer=Adam() # Initialize a variable to keep track of the batch count self.bc=tf.Variable(0, dtype=tf.float32) @@ -43,4 +43,4 @@ def loss(self, output, labels): def opt(self,gradient): # Perform optimization on the parameters using the gradient and the batch count param=self.optimizer.opt(gradient,self.param,self.bc) - return param + return param \ No newline at end of file diff --git a/Note 7.0 pv documentation/DL/neural network/tensorflow/process/attn_LSTM.py b/Note 7.0 pv documentation/DL/neural network/tensorflow/process/attn_LSTM.py index 728afb96..9979e726 100644 --- a/Note 7.0 pv documentation/DL/neural network/tensorflow/process/attn_LSTM.py +++ b/Note 7.0 pv documentation/DL/neural network/tensorflow/process/attn_LSTM.py @@ -2,14 +2,14 @@ from Note.nn.layer.LSTM import LSTM from Note.nn.layer.dense import dense from Note.nn.layer.attention import attention -import Note.nn.process.optimizer as o +from Note.nn.process.optimizer import Adam # Define a recurrent neural network class with LSTM layers and attention class attn_lstm: def __init__(self): # Initialize the loss function and the optimizer self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.optimizer=o.Adam() + self.optimizer=Adam() # Initialize a variable to keep track of the batch count self.bc=tf.Variable(0, dtype=tf.float32) diff --git a/Note 7.0 pv documentation/DL/neural network/tensorflow/process/cnn.py b/Note 7.0 pv documentation/DL/neural network/tensorflow/process/cnn.py index d5432232..7696e2f0 100644 --- a/Note 7.0 pv documentation/DL/neural network/tensorflow/process/cnn.py +++ b/Note 7.0 pv documentation/DL/neural network/tensorflow/process/cnn.py @@ -2,14 +2,14 @@ from Note.nn.layer.conv2d import conv2d from Note.nn.layer.dense import dense from Note.nn.layer.flatten import flatten -import Note.nn.process.optimizer as o +from Note.nn.process.optimizer import Adam # Define a convolutional neural network class class cnn: def __init__(self): # Initialize the loss function and the optimizer self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.optimizer=o.Adam() + self.optimizer=Adam() # Initialize a variable to keep track of the batch count self.bc=tf.Variable(0,dtype=tf.float32) @@ -56,4 +56,4 @@ def loss(self,output,labels): def opt(self,gradient): # Perform optimization on the parameters using the gradient and the batch count param=self.optimizer.opt(gradient,self.param,self.bc) - return param + return param \ No newline at end of file diff --git a/Note 7.0 pv documentation/DL/neural network/tensorflow/process/cnn_lmix.py b/Note 7.0 pv documentation/DL/neural network/tensorflow/process/cnn_lmix.py index 1883f175..9e7cd417 100644 --- a/Note 7.0 pv documentation/DL/neural network/tensorflow/process/cnn_lmix.py +++ b/Note 7.0 pv documentation/DL/neural network/tensorflow/process/cnn_lmix.py @@ -2,7 +2,7 @@ from Note.nn.layer.conv2d import conv2d from Note.nn.layer.dense import dense from Note.nn.layer.flatten import flatten -import Note.nn.process.optimizer as o +from Note.nn.process.optimizer import Adam from Note.nn.layer.LMix import lmix # Define a convolutional neural network class with mixup augmentation @@ -10,7 +10,7 @@ class cnn: def __init__(self): # Initialize the loss function and the optimizer self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.optimizer=o.Adam() + self.optimizer=Adam() # Initialize the alpha parameter for mixup self.alpha=1.0 # Initialize a variable to keep track of the batch count @@ -64,4 +64,4 @@ def loss(self,output,labels): def opt(self,gradient): # Perform optimization on the parameters using the gradient and the batch count param=self.optimizer.opt(gradient,self.param,self.bc) - return param + return param \ No newline at end of file diff --git a/Note 7.0 pv documentation/DL/neural network/tensorflow/process/nn.py b/Note 7.0 pv documentation/DL/neural network/tensorflow/process/nn.py index 24c65772..70f2a637 100644 --- a/Note 7.0 pv documentation/DL/neural network/tensorflow/process/nn.py +++ b/Note 7.0 pv documentation/DL/neural network/tensorflow/process/nn.py @@ -1,6 +1,6 @@ import tensorflow as tf import Note.nn.layer.dense as d -import Note.nn.process.optimizer as o +from Note.nn.process.optimizer import Momentum from Note.nn.layer.flatten import flatten # Define a neural network class @@ -8,7 +8,7 @@ class nn: def __init__(self): # Initialize the loss function and the optimizer self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.optimizer=o.Momentum(0.07,0.7) + self.optimizer=Momentum(0.07,0.7) def build(self): @@ -36,4 +36,4 @@ def loss(self,output,labels): def opt(self,gradient): # Perform optimization on the parameters using the gradient param=self.optimizer.opt(gradient,self.param) - return param + return param \ No newline at end of file diff --git a/Note 7.0 pv documentation/DL/neural network/tensorflow/process/nn_.py b/Note 7.0 pv documentation/DL/neural network/tensorflow/process/nn_.py index b1e11193..3f50c1e3 100644 --- a/Note 7.0 pv documentation/DL/neural network/tensorflow/process/nn_.py +++ b/Note 7.0 pv documentation/DL/neural network/tensorflow/process/nn_.py @@ -1,5 +1,5 @@ import tensorflow as tf -import Note.nn.process.optimizer as o +from Note.nn.process.optimizer import Momentum from Note.nn.layer.flatten import flatten # Define a neural network class @@ -16,7 +16,7 @@ def __init__(self): self.param=[self.weight1,self.weight2,self.weight3,self.bias1,self.bias2,self.bias3] # Initialize the loss function and the optimizer self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.optimizer=o.Momentum(0.07,0.7) + self.optimizer=Momentum(0.07,0.7) def fp(self,data): @@ -36,4 +36,4 @@ def loss(self,output,labels): def opt(self,gradient): # Perform optimization on the parameters using the gradient param=self.optimizer.opt(gradient,self.param) - return param + return param \ No newline at end of file diff --git a/Note 7.0 pv documentation/DL/neural network/tensorflow/process/nn_acc.py b/Note 7.0 pv documentation/DL/neural network/tensorflow/process/nn_acc.py index c33b7795..65311a69 100644 --- a/Note 7.0 pv documentation/DL/neural network/tensorflow/process/nn_acc.py +++ b/Note 7.0 pv documentation/DL/neural network/tensorflow/process/nn_acc.py @@ -1,6 +1,6 @@ import tensorflow as tf import Note.nn.layer.dense as d -import Note.nn.process.optimizer as o +from Note.nn.process.optimizer import Momentum from Note.nn.layer.flatten import flatten # Define a neural network class @@ -9,7 +9,7 @@ def __init__(self): # Initialize the loss function and the optimizer self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) self.train_accuracy=tf.keras.metrics.SparseCategoricalAccuracy(name='train_accuracy') - self.optimizer=o.Momentum(0.07,0.7) + self.optimizer=Momentum(0.07,0.7) def build(self): diff --git a/Note 7.0 pv documentation/DL/neural network/tensorflow/process/nn_attenuate.py b/Note 7.0 pv documentation/DL/neural network/tensorflow/process/nn_attenuate.py index 17e827fa..890b3335 100644 --- a/Note 7.0 pv documentation/DL/neural network/tensorflow/process/nn_attenuate.py +++ b/Note 7.0 pv documentation/DL/neural network/tensorflow/process/nn_attenuate.py @@ -1,6 +1,6 @@ import tensorflow as tf import Note.nn.layer.dense as d -import Note.nn.process.optimizer as o +from Note.nn.process.optimizer import Momentum from Note.nn.layer.flatten import flatten # Define a neural network class with gradient attenuation @@ -8,7 +8,7 @@ class nn: def __init__(self): # Initialize the loss function and the optimizer self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.optimizer=o.Momentum(0.07,0.7) + self.optimizer=Momentum(0.07,0.7) # Initialize a variable to keep track of the number of optimization steps self.opt_counter=tf.Variable(tf.zeros(7,dtype=tf.float32)) @@ -46,4 +46,4 @@ def attenuate(self,gradient,oc,p): #gradient attenuation function,kernel uses i def opt(self,gradient): # Perform optimization on the parameters using the gradient param=self.optimizer.opt(gradient,self.param) - return param + return param \ No newline at end of file diff --git a/Note 7.0 pv documentation/DL/neural network/tensorflow/process/nn_clipping.py b/Note 7.0 pv documentation/DL/neural network/tensorflow/process/nn_clipping.py index eb4d374b..32779f84 100644 --- a/Note 7.0 pv documentation/DL/neural network/tensorflow/process/nn_clipping.py +++ b/Note 7.0 pv documentation/DL/neural network/tensorflow/process/nn_clipping.py @@ -1,6 +1,6 @@ import tensorflow as tf import Note.nn.layer.dense as d -import Note.nn.process.optimizer as o +from Note.nn.process.optimizer import Momentum from Note.nn.layer.flatten import flatten from Note.nn.gradient_clipping import gradient_clipping @@ -9,7 +9,7 @@ class nn: def __init__(self): # Initialize the loss function and the optimizer self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.optimizer=o.Momentum(0.07,0.7) + self.optimizer=Momentum(0.07,0.7) def build(self): @@ -44,4 +44,4 @@ def gradient(self,tape,loss): def opt(self,gradient): # Perform optimization on the parameters using the gradient param=self.optimizer.opt(gradient,self.param) - return param + return param \ No newline at end of file diff --git a/Note 7.0 pv documentation/DL/neural network/tensorflow/process/self_LSTM.py b/Note 7.0 pv documentation/DL/neural network/tensorflow/process/self_LSTM.py index ea87f392..5a0d324a 100644 --- a/Note 7.0 pv documentation/DL/neural network/tensorflow/process/self_LSTM.py +++ b/Note 7.0 pv documentation/DL/neural network/tensorflow/process/self_LSTM.py @@ -1,5 +1,5 @@ import tensorflow as tf -import Note.nn.process.optimizer as o +from Note.nn.process.optimizer import Adam from Note.nn.layer.LSTM import LSTM from Note.nn.layer.self_attention import self_attention from Note.nn.layer.dense import dense @@ -13,7 +13,7 @@ def __init__(self,vocab_size,embed_size,num_heads,num_layers): self.num_heads=num_heads # Initialize the loss function and the optimizer self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.optimizer=o.Adam() + self.optimizer=Adam() # Initialize a variable to keep track of the batch count self.bc=tf.Variable(0,dtype=tf.float32) @@ -51,4 +51,4 @@ def loss(self,output,labels): def opt(self,gradient): # Perform optimization on the parameters using the gradient and the batch count param=self.optimizer.opt(gradient,self.param,self.bc) - return param + return param \ No newline at end of file diff --git a/Note 7.0 pv documentation/DL/neural network/tensorflow/process/transformer.py b/Note 7.0 pv documentation/DL/neural network/tensorflow/process/transformer.py new file mode 100644 index 00000000..b96a888a --- /dev/null +++ b/Note 7.0 pv documentation/DL/neural network/tensorflow/process/transformer.py @@ -0,0 +1,55 @@ +import tensorflow as tf +from Note.nn.process.optimizer import Adam +from Note.nn.layer.transformer import transformer +from Note.nn.layer.dense import dense +from Note.nn.layer.emdedding import embedding +from Note.nn.positional_encoding import positional_encoding + +# Define a positional encoding transformer neural network class +class Transformer: + def __init__(self,vocab_size,embed_size,num_heads,num_layers,max_len): + self.vocab_size=vocab_size + self.embed_size=embed_size + self.num_heads=num_heads + self.num_layers=num_layers + self.max_len=max_len + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.optimizer=Adam() + # Initialize a variable to keep track of the batch count + self.bc=tf.Variable(0,dtype=tf.float32) + + + def build(self): + # Create an embedding layer to map input tokens to vectors + self.embedding_layer = embedding(self.vocab_size,self.embed_size) + # Create a list of transformer layers + self.transformer_layers = [transformer(weight_shape=[self.embed_size, self.embed_size], num_heads=self.num_heads) for _ in range(self.num_layers)] + # Create a linear layer to map the transformer output to logits + self.logits_layer = dense([self.embed_size,self.vocab_size]) + # Store the parameters of the layers in a list + self.param = [self.embedding_layer.embeddings] + for transformer_layer in self.transformer_layers: + self.param.extend(transformer_layer.param) + self.param.extend(self.logits_layer.param) + + + def fp(self,data): + # Perform forward propagation on the input data + x = self.embedding_layer.output(data) # shape: (batch_size, seq_len, embed_size) + x += positional_encoding(self.max_len,self.embed_size) # shape: (batch_size, seq_len, embed_size) + for transformer_layer in self.transformer_layers: + x = transformer_layer.output(x) # shape: (batch_size, seq_len, embed_size) + output = self.logits_layer.output(x) # shape: (batch_size, seq_len, vocab_size) + return output + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) + + + def opt(self,gradient): + # Perform optimization on the parameters using the gradient and the batch count + param=self.optimizer.opt(gradient,self.param,self.bc) + return param \ No newline at end of file From 661093b1f128dd2bc154038ebcab837906c6e6d6 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Sat, 24 Jun 2023 18:18:26 +0800 Subject: [PATCH 002/337] Add files via upload --- .../DL/neural network/tensorflow/process/LSTM.py | 2 +- .../DL/neural network/tensorflow/process/attn_LSTM.py | 2 +- .../DL/neural network/tensorflow/process/cnn.py | 2 +- .../DL/neural network/tensorflow/process/cnn_lmix.py | 2 +- .../DL/neural network/tensorflow/process/self_LSTM.py | 2 +- .../DL/neural network/tensorflow/process/transformer.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Note 7.0 pv documentation/DL/neural network/tensorflow/process/LSTM.py b/Note 7.0 pv documentation/DL/neural network/tensorflow/process/LSTM.py index 3e0647f3..93ac8bf0 100644 --- a/Note 7.0 pv documentation/DL/neural network/tensorflow/process/LSTM.py +++ b/Note 7.0 pv documentation/DL/neural network/tensorflow/process/LSTM.py @@ -42,5 +42,5 @@ def loss(self, output, labels): def opt(self,gradient): # Perform optimization on the parameters using the gradient and the batch count - param=self.optimizer.opt(gradient,self.param,self.bc) + param=self.optimizer.opt(gradient,self.param,self.bc[0]) return param \ No newline at end of file diff --git a/Note 7.0 pv documentation/DL/neural network/tensorflow/process/attn_LSTM.py b/Note 7.0 pv documentation/DL/neural network/tensorflow/process/attn_LSTM.py index 9979e726..2326ad57 100644 --- a/Note 7.0 pv documentation/DL/neural network/tensorflow/process/attn_LSTM.py +++ b/Note 7.0 pv documentation/DL/neural network/tensorflow/process/attn_LSTM.py @@ -49,5 +49,5 @@ def loss(self,output,labels): def opt(self,gradient): # Perform optimization on the parameters using the gradient and the batch count - param=self.optimizer.opt(gradient,self.param,self.bc) + param=self.optimizer.opt(gradient,self.param,self.bc[0]) return param \ No newline at end of file diff --git a/Note 7.0 pv documentation/DL/neural network/tensorflow/process/cnn.py b/Note 7.0 pv documentation/DL/neural network/tensorflow/process/cnn.py index 7696e2f0..49892ccc 100644 --- a/Note 7.0 pv documentation/DL/neural network/tensorflow/process/cnn.py +++ b/Note 7.0 pv documentation/DL/neural network/tensorflow/process/cnn.py @@ -55,5 +55,5 @@ def loss(self,output,labels): def opt(self,gradient): # Perform optimization on the parameters using the gradient and the batch count - param=self.optimizer.opt(gradient,self.param,self.bc) + param=self.optimizer.opt(gradient,self.param,self.bc[0]) return param \ No newline at end of file diff --git a/Note 7.0 pv documentation/DL/neural network/tensorflow/process/cnn_lmix.py b/Note 7.0 pv documentation/DL/neural network/tensorflow/process/cnn_lmix.py index 9e7cd417..c1fa4cc9 100644 --- a/Note 7.0 pv documentation/DL/neural network/tensorflow/process/cnn_lmix.py +++ b/Note 7.0 pv documentation/DL/neural network/tensorflow/process/cnn_lmix.py @@ -63,5 +63,5 @@ def loss(self,output,labels): def opt(self,gradient): # Perform optimization on the parameters using the gradient and the batch count - param=self.optimizer.opt(gradient,self.param,self.bc) + param=self.optimizer.opt(gradient,self.param,self.bc[0]) return param \ No newline at end of file diff --git a/Note 7.0 pv documentation/DL/neural network/tensorflow/process/self_LSTM.py b/Note 7.0 pv documentation/DL/neural network/tensorflow/process/self_LSTM.py index 5a0d324a..a25f4043 100644 --- a/Note 7.0 pv documentation/DL/neural network/tensorflow/process/self_LSTM.py +++ b/Note 7.0 pv documentation/DL/neural network/tensorflow/process/self_LSTM.py @@ -50,5 +50,5 @@ def loss(self,output,labels): def opt(self,gradient): # Perform optimization on the parameters using the gradient and the batch count - param=self.optimizer.opt(gradient,self.param,self.bc) + param=self.optimizer.opt(gradient,self.param,self.bc[0]) return param \ No newline at end of file diff --git a/Note 7.0 pv documentation/DL/neural network/tensorflow/process/transformer.py b/Note 7.0 pv documentation/DL/neural network/tensorflow/process/transformer.py index b96a888a..a77978a6 100644 --- a/Note 7.0 pv documentation/DL/neural network/tensorflow/process/transformer.py +++ b/Note 7.0 pv documentation/DL/neural network/tensorflow/process/transformer.py @@ -51,5 +51,5 @@ def loss(self,output,labels): def opt(self,gradient): # Perform optimization on the parameters using the gradient and the batch count - param=self.optimizer.opt(gradient,self.param,self.bc) + param=self.optimizer.opt(gradient,self.param,self.bc[0]) return param \ No newline at end of file From dff1d8137588e829bc167a7497a7b029d31e53a2 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Sun, 25 Jun 2023 17:56:08 +0800 Subject: [PATCH 003/337] Update README.md --- README.md | 74 ++++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 62 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 00233a70..5baf1acc 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ kernel=k.kernel(nn) #start kernel kernel.platform=tf #use platform kernel.stop=True kernel.end_loss=0.7 -kernel.data(x_train,y_train) #input you data +kernel.data(x_train,y_train) #input train data kernel.train(32,5) #train neural network #batch size:32 #epoch:5 @@ -56,7 +56,7 @@ kernel=k.kernel(nn) #start kernel kernel.threading=threading kernel.thread_t=6 #test thread count kernel.platform=tf #use platform -kernel.data(x_train,y_train,x_test,y_test) #input you data +kernel.data(x_train,y_train,x_test,y_test) #input train data kernel.train(32,5,32) #train neural network #batch size:32 #test batch size:32 @@ -107,7 +107,7 @@ kernel.data_segment_flag=True kernel.epoch=6 #epoch:6 kernel.batch=32 #batch:32 kernel.PO=3 #use PO3 -kernel.data(x_train,y_train) #input you data +kernel.data(x_train,y_train) #input train data manager=Manager() #create manager object kernel.init(manager) #initialize shared data lock=[Lock(),Lock()] @@ -138,7 +138,7 @@ kernel.data_segment_flag=True kernel.epoch=6 #epoch:6 kernel.batch=32 #batch:32 kernel.PO=3 #use PO3 -kernel.data(x_train,y_train) #input you data +kernel.data(x_train,y_train) #input train data manager=Manager() #create manager object kernel.init(manager) #initialize shared data lock=[Lock(),Lock()] @@ -146,6 +146,56 @@ g_lock=[Lock(),Lock(),Lock()] for p in range(7): Process(target=kernel.train,args=(p,lock,g_lock)).start() ``` +## PO4： +```python +import Note.DL.process.kernel as k #import kernel +import tensorflow as tf +import nn as n #import neural network +from multiprocessing import Process,Lock,Manager +mnist=tf.keras.datasets.mnist +(x_train,y_train),(x_test,y_test)=mnist.load_data() +x_train,x_test =x_train/255.0,x_test/255.0 +nn=n.nn() #create neural network object +nn.build() +kernel=k.kernel(nn) #start kernel +kernel.process=7 #7 processes to train +kernel.data_segment_flag=True +kernel.epoch=6 #epoch:6 +kernel.batch=32 #batch:32 +kernel.PO=4 #use PO4 +kernel.data(x_train,y_train) #input train data +manager=Manager() #create manager object +kernel.init(manager) #initialize shared data +lock=Lock() +for p in range(7): + Process(target=kernel.train,args=(p,lock)).start() +kernel.update_nn_param() +kernel.test(x_train,y_train,32) +``` +```python +import Note.DL.process.kernel as k #import kernel +import tensorflow as tf +import nn as n #import neural network +from multiprocessing import Process,Lock,Manager +mnist=tf.keras.datasets.mnist +(x_train,y_train),(x_test,y_test)=mnist.load_data() +x_train,x_test =x_train/255.0,x_test/255.0 +nn=n.nn() #create neural network object +nn.build() +kernel=k.kernel(nn) #start kernel +kernel.process=7 #7 processes to train +kernel.data_segment_flag=True +kernel.epoch=6 #epoch:6 +kernel.batch=32 #batch:32 +kernel.PO=4 #use PO4 +kernel.data(x_train,y_train) #input train data +manager=Manager() #create manager object +kernel.init(manager) #initialize shared data +for p in range(7): + Process(target=kernel.train,args=(p,)).start() +kernel.update_nn_param() +kernel.test(x_train,y_train,32) +``` # Multithreading: @@ -169,7 +219,7 @@ kernel.platform=tf #use platform kernel.thread=7 #thread count,use 7 threads to train kernel.epoch_=6 #epoch:6 kernel.PO=2 #use PO2 -kernel.data(x_train,y_train) #input you data +kernel.data(x_train,y_train) #input train data kernel.lock=[threading.Lock(),threading.Lock(),threading.Lock()] class thread(threading.Thread): def run(self): @@ -199,7 +249,7 @@ kernel.data_segment_flag=True kernel.batches=1875 #batches:1875 kernel.epoch_=6 #epoch:6 kernel.PO=2 -kernel.data(x_train,y_train) #input you data +kernel.data(x_train,y_train) #input train data kernel.lock=[threading.Lock(),threading.Lock(),threading.Lock()] class thread(threading.Thread): def run(self): @@ -226,7 +276,7 @@ kernel=k.kernel(nn) #start kernel kernel.platform=tf #use platform kernel.thread=2 #thread count,use 2 threads to train kernel.PO=2 #use PO2 -kernel.data(x_train,y_train) #input you data +kernel.data(x_train,y_train) #input train data kernel.lock=[threading.Lock(),threading.Lock(),threading.Lock()] class thread(threading.Thread): def run(self): @@ -256,7 +306,7 @@ kernel.stop=True kernel.end_loss=0.7 #stop condition kernel.thread=2 #thread count,use 2 threads to train kernel.PO=2 -kernel.data(x_train,y_train) #input you data +kernel.data(x_train,y_train) #input train data kernel.lock=[threading.Lock(),threading.Lock(),threading.Lock()] class thread(threading.Thread): def run(self): @@ -351,7 +401,7 @@ kernel.thread=7 #thread count,use 7 threads to train kernel.PO=3 #use PO3 kernel.threading=threading kernel.max_lock=7 -kernel.data(x_train,y_train) #input you data +kernel.data(x_train,y_train) #input train data kernel.lock=[threading.Lock(),threading.Lock()] class thread(threading.Thread): def run(self): @@ -385,7 +435,7 @@ kernel.thread=7 #thread count,use 7 threads to train kernel.thread_t=6 #test thread count kernel.epoch_=6 #epoch:6 kernel.PO=2 -kernel.data(x_train,y_train,x_test,y_test) #input you data +kernel.data(x_train,y_train,x_test,y_test) #input train data kernel.lock=[threading.Lock(),threading.Lock(),threading.Lock()] class thread(threading.Thread): def run(self): @@ -417,7 +467,7 @@ kernel=k.kernel(nn) #start kernel kernel.platform=tf #use platform kernel.thread=7 #thread count,use 7 thread to train kernel.PO=2 -kernel.create_t_num(7) #input you data +kernel.create_t_num(7) #input train data kernel.lock=[threading.Lock(),threading.Lock(),threading.Lock()] class thread(threading.Thread): def run(self): @@ -470,7 +520,7 @@ x_train,x_test =x_train/255.0,x_test/255.0 nn=n.nn(x_train,y_train) #create neural network object kernel=k.kernel(nn) #start kernel kernel.platform=tf #use platform -kernel.data(x_train,y_train) #input you data +kernel.data(x_train,y_train) #input train data kernel.train_ol() #train neural network ``` From 61efaf309e960157b75ddef716b340714c9aca51 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Tue, 27 Jun 2023 13:47:27 +0800 Subject: [PATCH 004/337] Update README.md --- README.md | 41 ++++++++++++++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 5baf1acc..a3fc53b5 100644 --- a/README.md +++ b/README.md @@ -88,8 +88,8 @@ loss,acc=test.loss_acc() # Multiprocessing: -## PO3: -**multiprocessing example(PO3):** +## PO2: +**multiprocessing example(PO2):** ```python import Note.DL.process.kernel as k #import kernel import tensorflow as tf @@ -106,7 +106,7 @@ kernel.process=7 #7 processes to train kernel.data_segment_flag=True kernel.epoch=6 #epoch:6 kernel.batch=32 #batch:32 -kernel.PO=3 #use PO3 +kernel.PO=2 #use PO3 kernel.data(x_train,y_train) #input train data manager=Manager() #create manager object kernel.init(manager) #initialize shared data @@ -117,6 +117,33 @@ for p in range(7): kernel.update_nn_param() kernel.test(x_train,y_train,32) ``` +```python +import Note.DL.process.kernel as k #import kernel +import tensorflow as tf +import nn as n #import neural network +from multiprocessing import Process,Lock,Manager +mnist=tf.keras.datasets.mnist +(x_train,y_train),(x_test,y_test)=mnist.load_data() +x_train,x_test =x_train/255.0,x_test/255.0 +x_train=x_train.reshape([60000,784]) +nn=n.nn() #create neural network object +nn.build() +kernel=k.kernel(nn) #start kernel +kernel.process=7 #7 processes to train +kernel.data_segment_flag=True +kernel.epoch=6 #epoch:6 +kernel.batch=32 #batch:32 +kernel.PO=2 #use PO3 +kernel.data(x_train,y_train) #input train data +manager=Manager() #create manager object +kernel.init(manager) #initialize shared data +lock=[Lock(),Lock()] +g_lock=Lock() +for p in range(7): + Process(target=kernel.train,args=(p,lock,g_lock)).start() +kernel.update_nn_param() +kernel.test(x_train,y_train,32) +``` ## Stop training and saving when condition is met: **multiprocessing example(Stop training and saving when condition is met.):** ```python @@ -137,7 +164,7 @@ kernel.process=7 #7 processes to train kernel.data_segment_flag=True kernel.epoch=6 #epoch:6 kernel.batch=32 #batch:32 -kernel.PO=3 #use PO3 +kernel.PO=2 #use PO3 kernel.data(x_train,y_train) #input train data manager=Manager() #create manager object kernel.init(manager) #initialize shared data @@ -146,7 +173,7 @@ g_lock=[Lock(),Lock(),Lock()] for p in range(7): Process(target=kernel.train,args=(p,lock,g_lock)).start() ``` -## PO4： +## PO3： ```python import Note.DL.process.kernel as k #import kernel import tensorflow as tf @@ -162,7 +189,7 @@ kernel.process=7 #7 processes to train kernel.data_segment_flag=True kernel.epoch=6 #epoch:6 kernel.batch=32 #batch:32 -kernel.PO=4 #use PO4 +kernel.PO=3 #use PO4 kernel.data(x_train,y_train) #input train data manager=Manager() #create manager object kernel.init(manager) #initialize shared data @@ -187,7 +214,7 @@ kernel.process=7 #7 processes to train kernel.data_segment_flag=True kernel.epoch=6 #epoch:6 kernel.batch=32 #batch:32 -kernel.PO=4 #use PO4 +kernel.PO=3 #use PO4 kernel.data(x_train,y_train) #input train data manager=Manager() #create manager object kernel.init(manager) #initialize shared data From 079cd399abf47c8a43afbd5b18e26580be966ad7 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Wed, 28 Jun 2023 00:09:17 +0800 Subject: [PATCH 005/337] Update kernel_reduced.py --- reduced kernel/DL/process/kernel_reduced.py | 1092 +++++++++---------- 1 file changed, 534 insertions(+), 558 deletions(-) diff --git a/reduced kernel/DL/process/kernel_reduced.py b/reduced kernel/DL/process/kernel_reduced.py index c5bdc4b5..8ed97bfe 100644 --- a/reduced kernel/DL/process/kernel_reduced.py +++ b/reduced kernel/DL/process/kernel_reduced.py @@ -1,657 +1,633 @@ import tensorflow as tf -from tensorflow import function from tensorflow.python.ops import state_ops +from tensorflow.python.util import nest from multiprocessing import Value,Array import numpy as np import matplotlib.pyplot as plt -import pickle -import os class kernel: def __init__(self,nn=None): - self.nn=nn #Neural network object. - self.PO=None #PO object,two parallel optimization methods correspond to three numbers. - self.process=None #processes count you use. - self.train_ds=None - self.data_segment_flag=False - self.batches=None - self.buffer_size=None - self.epoch=None #Training epoch count. - self.epoch_counter=0 - self.stop=False - self.stop_flag=False - self.save_flag=False - self.save_epoch=None - self.batch=None - self.epoch_=0 - self.end_loss=None - self.end_acc=None - self.end_test_loss=None - self.end_test_acc=None - self.acc_flag='%' #This object be used for acc printing. - self.train_counter=0 - self.muti_p=None - self.muti_s=None - self.muti_save=1 - self.filename='save.dat' - self.train_loss=None - self.train_acc=None - self.train_loss_list=[] #This object be used for visualization function. - self.train_acc_list=[] #This object be used for visualization function. - self.test_loss=None - self.test_acc=None - self.test_loss_list=[] #This object be used for visualization function. - self.test_acc_list=[] #This object be used for visualization function. - self.test_flag=False #If you have test data,kernel will set it to True. - self.total_epoch=0 + self.nn=nn # the neural network object + try: + self.nn.km=1 # a flag to indicate the kernel mode + except Exception: + pass + self.PO=None # the order of the optimizer + self.process=None # the number of processes + self.train_ds=None # the training dataset + self.data_segment_flag=False # a flag to indicate whether to segment the data + self.batches=None # the number of batches + self.buffer_size=None # the buffer size for shuffling the data + self.priority_flag=False # a flag to indicate whether to use priority for optimization + self.priority_p=0 # the priority parameter + self.max_opt=None # the maximum number of optimization steps + self.epoch=None # the number of epochs + self.epoch_counter=0 # the epoch counter + self.stop=False # a flag to indicate whether to stop the training + self.stop_flag=False # a flag to indicate whether to stop the training by condition + self.save_flag=False # a flag to indicate whether to save the model + self.batch=None # the batch size + self.end_loss=None # the end condition for training loss + self.end_acc=None # the end condition for training accuracy + self.end_test_loss=None # the end condition for test loss + self.end_test_acc=None # the end condition for test accuracy + self.acc_flag='%' # the format for displaying accuracy + self.opt_counter=None # the counter for optimization steps + self.train_loss=0 # the training loss + self.train_acc=0 # the training accuracy + self.train_loss_list=[] # the list of training loss values + self.train_acc_list=[] # the list of training accuracy values + self.test_loss=0 # the test loss + self.test_acc=0 # the test accuracy + self.test_loss_list=[] # the list of test loss values + self.test_acc_list=[] # the list of test accuracy values + self.test_flag=False # a flag to indicate whether to use test data + self.total_epoch=0 # the total number of epochs - #Turn training data into kernel's instance object and initialize some objects. def data(self,train_data=None,train_labels=None,test_data=None,test_labels=None,train_dataset=None,test_dataset=None): - self.train_data=train_data - self.train_labels=train_labels - self.train_dataset=train_dataset - self.test_data=test_data - self.test_labels=test_labels - self.test_dataset=test_dataset - try: #If have test data,kernel set test_flag=True. - if test_data==None: - self.test_flag=False - except ValueError: - self.test_flag=True - self.batch_counter=np.zeros(self.process,dtype=np.int32) - self.total_loss=np.zeros(self.process,dtype=np.float32) + if type(self.nn.param[0])!=list: + self.train_data=train_data.astype(self.nn.param[0].dtype.name) # convert the train data type to match the model parameter type + self.train_labels=train_labels.astype(self.nn.param[0].dtype.name) # convert the train labels type to match the model parameter type + else: + self.train_data=train_data.astype(self.nn.param[0][0].dtype.name) # convert the train data type to match the model parameter type + self.train_labels=train_labels.astype(self.nn.param[0][0].dtype.name) # convert the train labels type to match the model parameter type + self.train_dataset=train_dataset # set the train dataset object + if test_data is not None: + self.test_data=test_data # set the test data array + self.test_labels=test_labels # set the test labels array + self.test_flag=True # set the test flag to True + self.test_dataset=test_dataset # set the test dataset object + self.batch_counter=np.zeros(self.process,dtype=np.int32) # initialize an array to count batches for each process + self.total_loss=np.zeros(self.process,dtype=np.float32) # initialize an array to accumulate loss for each process + try: + if self.nn.accuracy!=None: + self.total_acc=np.zeros(self.process,dtype=np.float32) # initialize an array to accumulate accuracy for each process + except Exception: + pass + if self.priority_flag==True: + self.opt_counter=np.zeros(self.process,dtype=np.int32) # initialize an array to count optimization steps for each process if self.train_dataset==None: if type(self.train_data)==list: - self.shape0=train_data[0].shape[0] - self.batches=int((self.shape0-self.shape0%self.batch)/self.batch) + self.shape0=train_data[0].shape[0] # get the number of samples in the first train data array + self.batches=int((self.shape0-self.shape0%self.batch)/self.batch) # calculate the number of batches if self.shape0%self.batch!=0: - self.batches+=1 + self.batches+=1 # add one more batch if there are remaining samples else: - self.shape0=train_data.shape[0] - self.batches=int((self.shape0-self.shape0%self.batch)/self.batch) + self.shape0=train_data.shape[0] # get the number of samples in the train data array + self.batches=int((self.shape0-self.shape0%self.batch)/self.batch) # calculate the number of batches if self.shape0%self.batch!=0: - self.batches+=1 + self.batches+=1 # add one more batch if there are remaining samples if self.data_segment_flag==True: - self.train_data,self.train_labels=self.segment_data() + self.train_data,self.train_labels=self.segment_data() # segment the train data and labels according to the number of processes return def segment_data(self): - if len(self.train_data)!=self.process: + if len(self.train_data)!=self.process: # check if the train data is already segmented data=None labels=None - segments=int((len(self.train_data)-len(self.train_data)%self.process)/self.process) - for i in range(self.process): - index1=i*segments - index2=(i+1)*segments - if i==0: - data=np.expand_dims(self.train_data[index1:index2],axis=0) - labels=np.expand_dims(self.train_labels[index1:index2],axis=0) - else: - data=np.concatenate((data,np.expand_dims(self.train_data[index1:index2],axis=0))) - labels=np.concatenate((labels,np.expand_dims(self.train_labels[index1:index2],axis=0))) - if len(data)%self.process!=0: - segments+=1 - index1=segments*self.process - index2=self.process-(len(self.train_data)-segments*self.process) - data=np.concatenate((data,np.expand_dims(self.train_data[index1:index2],axis=0))) - labels=np.concatenate((labels,np.expand_dims(self.train_labels[index1:index2],axis=0))) - return data,labels + segments=int((len(self.train_data)-len(self.train_data)%self.process)/self.process) # calculate the number of samples for each segment + for i in range(self.process): # loop over the processes + index1=i*segments # get the start index of the segment + index2=(i+1)*segments # get the end index of the segment + if i==0: # for the first process + data=np.expand_dims(self.train_data[index1:index2],axis=0) # create a new dimension for the segment and assign it to data + labels=np.expand_dims(self.train_labels[index1:index2],axis=0) # create a new dimension for the segment and assign it to labels + else: # for other processes + data=np.concatenate((data,np.expand_dims(self.train_data[index1:index2],axis=0))) # concatenate the segment to data along the new dimension + labels=np.concatenate((labels,np.expand_dims(self.train_labels[index1:index2],axis=0))) # concatenate the segment to labels along the new dimension + if len(data)%self.process!=0: # check if there are remaining samples that are not segmented + segments+=1 # increase the number of samples for each segment by one + index1=segments*self.process # get the start index of the remaining samples + index2=self.process-(len(self.train_data)-segments*self.process) # get the number of processes that need to be filled with extra samples + data=np.concatenate((data,np.expand_dims(self.train_data[index1:index2],axis=0))) # concatenate the remaining samples to data along the new dimension + labels=np.concatenate((labels,np.expand_dims(self.train_labels[index1:index2],axis=0))) # concatenate the remaining samples to labels along the new dimension + return data,labels # return the segmented data and labels - #Initialize shared data for process. + def init(self,manager): - self.epoch_counter=Value('i',self.epoch_counter) #epoch counter,train7 function returns when epoch counter is equal to epoch. - self.batch_counter=Array('i',self.batch_counter) #batch counter,train7 function will do some work when the batch counter sum is equal to batches. - self.total_loss=Array('f',self.total_loss) #For calculating loss. - self.total_epoch=Value('i',self.total_epoch) - self.train_loss=Value('f',self.train_loss) - self.train_loss_list=manager.list(self.train_loss_list) + self.epoch_counter=Value('i',self.epoch_counter) # create a shared value for epoch counter + self.batch_counter=Array('i',self.batch_counter) # create a shared array for batch counter + self.total_loss=Array('f',self.total_loss) # create a shared array for total loss + self.total_epoch=Value('i',self.total_epoch) # create a shared value for total epoch + self.train_loss=Value('f',self.train_loss) # create a shared value for train loss + self.train_loss_list=manager.list(self.train_loss_list) # create a shared list for train loss values + self.priority_p=Value('i',self.priority_p) # create a shared value for priority parameter if self.test_flag==True: - self.test_loss=Value('f',self.test_loss) - self.test_loss_list=manager.list(self.test_loss_list) + self.test_loss=Value('f',self.test_loss) # create a shared value for test loss + self.test_loss_list=manager.list(self.test_loss_list) # create a shared list for test loss values try: if self.nn.accuracy!=None: - self.train_acc=Value('f',self.train_acc) - self.train_acc_list=manager.list(self.train_acc_list) + self.total_acc=Array('f',self.total_acc) # create a shared array for total accuracy + self.train_acc=Value('f',self.train_acc) # create a shared value for train accuracy + self.train_acc_list=manager.list(self.train_acc_list) # create a shared list for train accuracy values if self.test_flag==True: - self.test_acc=Value('f',self.test_acc) - self.test_acc_list=manager.list(self.test_acc_list) - except AttributeError: + self.test_acc=Value('f',self.test_acc) # create a shared value for test accuracy + self.test_acc_list=manager.list(self.test_acc_list) # create a shared list for test accuracy values + except Exception: pass - self.stop_flag=Value('b',self.stop_flag) - self.save_flag=Value('b',self.save_flag) - self.file_list=manager.list([]) - self.param=manager.dict() #A dictionary to store optimized parameters. + if self.priority_flag==True: + self.opt_counter=Array('i',self.opt_counter) # create a shared array for optimization counter + try: + if self.nn.attenuate!=None: + self.nn.opt_counter=manager.list([self.nn.opt_counter]) # create a shared list for the neural network's optimization counter + except Exception: + pass + try: + self.nn.ec=manager.list([self.nn.ec]) # create a shared list for the neural network's epoch counter + except Exception: + pass + try: + self.nn.bc=manager.list([self.nn.bc]) # create a shared list for the neural network's batch counter + except Exception: + pass + self.stop_flag=Value('b',self.stop_flag) # create a shared value for stop flag + self.save_flag=Value('b',self.save_flag) # create a shared value for save flag + self.param=manager.dict() # create a shared dictionary for parameters + self.param[7]=self.nn.param # assign the neural network's parameters to the dictionary return def end(self): - if self.end_loss!=None and len(self.train_loss_list)!=0 and self.train_loss_list[-1]self.end_acc: + elif self.end_acc!=None and len(self.train_acc_list)!=0 and self.train_acc_list[-1]>self.end_acc: # check if the train accuracy is higher than the end condition return True - elif self.end_loss!=None and len(self.train_loss_list)!=0 and self.end_acc!=None and self.train_loss_list[-1]self.end_acc: + elif self.end_loss!=None and len(self.train_loss_list)!=0 and self.end_acc!=None and self.train_loss_list[-1]self.end_acc: # check if both the train loss and accuracy meet the end condition return True - elif self.end_test_loss!=None and len(self.test_loss_list)!=0 and self.test_loss_list[-1]self.end_test_acc: + elif self.end_test_acc!=None and len(self.test_acc_list)!=0 and self.test_acc_list[-1]>self.end_test_acc: # check if the test accuracy is higher than the end condition return True - elif self.end_test_loss!=None and len(self.test_loss_list)!=0 and self.end_test_acc!=None and self.test_loss_list[-1]self.end_test_acc: + elif self.end_test_loss!=None and len(self.test_loss_list)!=0 and self.end_test_acc!=None and self.test_loss_list[-1]self.end_test_acc: # check if both the test loss and accuracy meet the end condition return True - #Optimization subfunction,it be used for opt function,it used optimization function of tensorflow platform and parallel optimization. - @function(jit_compile=True) - def tf_opt_t(self,data,labels,p,lock): - try: #If neural network object define GradientTape function,kernel will use it or else use other,this design allow you to implement more complicated operations. - if self.nn.GradientTape!=None: - tape,output,loss=self.nn.GradientTape(data,labels,p) - except AttributeError: - with tf.GradientTape(persistent=True) as tape: + @tf.function + def opt_p(self,data,labels,p,lock,g_lock=None): + try: + try: + if self.nn.GradientTape!=None: # check if the neural network has its own gradient tape function + tape,output,loss=self.nn.GradientTape(data,labels,p) # use the neural network's gradient tape function to get the tape, output and loss + except Exception: + with tf.GradientTape(persistent=True) as tape: # use the default gradient tape function + try: + try: + output=self.nn.fp(data) # use the neural network's forward propagation function to get the output + loss=self.nn.loss(output,labels) # use the neural network's loss function to get the loss + except Exception: + output,loss=self.nn.fp(data,labels) # use the neural network's forward propagation function to get the output and loss + except Exception: + try: + output=self.nn.fp(data,p) # use the neural network's forward propagation function to get the output with the process number + loss=self.nn.loss(output,labels) # use the neural network's loss function to get the loss + except Exception: + output,loss=self.nn.fp(data,labels,p) # use the neural network's forward propagation function to get the output and loss with the process number + except Exception as e: + raise e + if self.PO==1: # check if the optimizer order is 1 (lock before gradient calculation) + if self.priority_flag==True and self.priority_p.value!=-1: # check if the priority flag is True and the priority parameter is not -1 + while True: + if p==self.priority_p.value: # check if the process number matches the priority parameter + break + else: + continue + lock[0].acquire() # acquire the first lock + if self.stop_func_(lock[0]): # check if the stop condition is met + return None,0 + try: try: - try: #If neural network object define one argument value fp(forward propagation) function,kernel will use it or else use other,this design allow you to implement more complicated operations. - output=self.nn.fp(data) - loss=self.nn.loss(output,labels) - except TypeError: - output,loss=self.nn.fp(data,labels) - except TypeError: try: - output=self.nn.fp(data,p) - loss=self.nn.loss(output,labels) - except TypeError: - output,loss=self.nn.fp(data,labels,p) - if self.PO==1: - lock[0].acquire() - if self.stop_func_(lock[0]): + gradient=self.nn.gradient(tape,loss) # use the neural network's gradient function to get the gradient + except Exception: + gradient=self.nn.gradient(tape,loss,self.param[7]) # use the neural network's gradient function to get the gradient with the parameters + except Exception: + gradient=tape.gradient(loss,self.nn.param) # use the default gradient function to get the gradient + except Exception as e: + raise e + try: + gradient=self.nn.attenuate(gradient,self.nn.opt_counter,p) # use the neural network's attenuate function to modify the gradient + except Exception as e: + try: + if self.nn.attenuate!=None: # check if the neural network has an attenuate function + raise e + except Exception: + pass + try: + try: + param=self.nn.opt(gradient) # use the neural network's optimizer function to update the parameters + except Exception: + param=self.nn.opt(gradient,p) # use the neural network's optimizer function to update the parameters with the process number + except Exception as e: + raise e + lock[0].release() # release the first lock + elif self.PO==2: # check if the optimizer order is 2 (lock after gradient calculation) + g_lock.acquire() # acquire the global lock + if self.stop_func_(g_lock): # check if the stop condition is met return None,0 - try: #If neural network object define gradient function,kernel will use it or else use other,this design allow you to implement more complicated operations. - gradient=self.nn.gradient(tape,loss) - except AttributeError: - gradient=tape.gradient(loss,self.nn.param) try: - param=self.nn.opt(gradient) - except TypeError: - param=self.nn.opt(gradient,p) - lock[0].release() - elif self.PO==2: - lock[0].acquire() - if self.stop_func_(lock[0]): + try: + try: + gradient=self.nn.gradient(tape,loss) # use the neural network's gradient function to get the gradient + except Exception: + gradient=self.nn.gradient(tape,loss,self.param[7]) # use the neural network's gradient function to get the gradient with the parameters + except Exception: + gradient=tape.gradient(loss,self.nn.param) # use the default gradient function to get the gradient + except Exception as e: + raise e + g_lock.release() # release the global lock + if self.priority_flag==True and self.priority_p.value!=-1: # check if the priority flag is True and the priority parameter is not -1 + while True: + if p==self.priority_p.value: # check if the process number matches the priority parameter + break + else: + continue + lock[0].acquire() # acquire the first lock + if self.stop_func_(lock[0]): # check if the stop condition is met return None,0 - try: #If neural network object define gradient function,kernel will use it or else use other,this design allow you to implement more complicated operations. - gradient=self.nn.gradient(tape,loss) - except AttributeError: - gradient=tape.gradient(loss,self.nn.param) - lock[0].release() - lock[1].acquire() - if self.stop_func_(lock[1]): + try: + gradient=self.nn.attenuate(gradient,self.nn.opt_counter,p) # use the neural network's attenuate function to modify the gradient + except Exception as e: + try: + if self.nn.attenuate!=None: # check if the neural network has an attenuate function + raise e + except Exception: + pass + try: + try: + param=self.nn.opt(gradient) # use the neural network's optimizer function to update the parameters + except Exception: + param=self.nn.opt(gradient,p) # use the neural network's optimizer function to update the parameters with the process number + except Exception as e: + raise e + lock[0].release() # release the first lock + elif self.PO==3: # check if the optimizer order is 3 (no lock) + if self.priority_flag==True and self.priority_p.value!=-1: # check if the priority flag is True and the priority parameter is not -1 + while True: + if p==self.priority_p.value: # check if the process number matches the priority parameter + break + else: + continue + if self.stop_func_(): # check if the stop condition is met return None,0 try: - param=self.nn.opt(gradient) - except TypeError: - param=self.nn.opt(gradient,p) - lock[1].release() - return output,loss,param + try: + try: + gradient=self.nn.gradient(tape,loss) # use the neural network's gradient function to get the gradient + except Exception: + gradient=self.nn.gradient(tape,loss,self.param[7]) # use the neural network's gradient function to get the gradient with the parameters + except Exception: + gradient=tape.gradient(loss,self.nn.param) # use the default gradient function to get the gradient + except Exception as e: + raise e + try: + gradient=self.nn.attenuate(gradient,self.nn.opt_counter,p) # use the neural network's attenuate function to modify the gradient + except Exception as e: + try: + if self.nn.attenuate!=None: # check if the neural network has an attenuate function + raise e + except Exception: + pass + try: + try: + param=self.nn.opt(gradient) # use the neural network's optimizer function to update the parameters + except Exception: + param=self.nn.opt(gradient,p) # use the neural network's optimizer function to update the parameters with the process number + except Exception as e: + raise e + return output,loss,param # return output, loss and parameters - #Main optimization function,it be used for parallel optimization. - def opt_t(self,data,labels,p,lock): - output,loss,param=self.tf_opt_t(data,labels,p,lock) - return output,loss,param + + def opt(self,data,labels,p,lock,g_lock): + if self.PO==2: # check if the optimizer order is 2 (lock after gradient calculation) + if type(g_lock)!=list: # check if g_lock is not a list + pass + elif len(g_lock)==self.process: # check if g_lock has same length as process number + ln=p # assign process number to ln (local number) + g_lock=g_lock[ln] # assign g_lock element at ln index to g_lock + else: + ln=int(np.random.choice(len(g_lock))) # assign a random integer from g_lock length to ln (local number) + g_lock=g_lock[ln] # assign g_lock element at ln index to g_lock + output,loss,param=self.opt_p(data,labels,p,lock,g_lock) # call the opt_p function to get output, loss and parameters + else: + output,loss,param=self.opt_p(data,labels,p,lock) # call the opt_p function to get output, loss and parameters without g_lock + return output,loss,param # return output, loss and parameters def update_nn_param(self,param=None): - for i in range(len(self.nn.param)): - if param==None: - self.param[7][i]=self.param[7][i].numpy() - state_ops.assign(self.nn.param[i],self.param[7][i]) + if param==None: # check if param is None + parameter_flat=nest.flatten(self.nn.param) # flatten the neural network's parameters + parameter7_flat=nest.flatten(self.param[7]) # flatten the kernel's parameters + else: + parameter_flat=nest.flatten(self.nn.param) # flatten the neural network's parameters + parameter7_flat=nest.flatten(param) # flatten the given param + for i in range(len(parameter_flat)): # loop over the flattened parameters + if param==None: # check if param is None + state_ops.assign(parameter_flat[i],parameter7_flat[i]) # assign the kernel's parameters to the neural network's parameters else: - param[7][i]=param[7][i].numpy() - state_ops.assign(self.nn.param[i],param[7][i]) + state_ops.assign(parameter_flat[i],parameter7_flat[i]) # assign the given param to the neural network's parameters + self.nn.param=nest.pack_sequence_as(self.nn.param,parameter_flat) # pack the flattened parameters back to the neural network's parameters + self.param[7]=nest.pack_sequence_as(self.param[7],parameter7_flat) # pack the flattened parameters back to the kernel's parameters return - #Training subfunction,it be used for train function and parallel training. - def train7(self,train_ds,p,test_batch,lock): - while True: - for data_batch,labels_batch in train_ds: - output,batch_loss,param=self.opt_t(data_batch,labels_batch,p,lock) - self.param[7]=param + + def train7(self,train_ds,p,test_batch,lock,g_lock): + while True: # loop until break + for data_batch,labels_batch in train_ds: # loop over the train dataset batches + try: + data_batch,labels_batch=self.nn.data_func(data_batch,labels_batch) # use the neural network's data function to process data and labels + except Exception as e: + try: + if self.nn.data_func!=None: # check if the neural network has a data function + raise e + except Exception: + pass + if self.priority_flag==True: # check if the priority flag is True + self.priority_p.value=np.argmax(self.opt_counter) # assign the index of the maximum value in opt_counter to priority parameter + if self.max_opt!=None and self.opt_counter[self.priority_p.value]>=self.max_opt: # check if max_opt is not None and opt_counter at priority parameter index is greater than or equal to max_opt + self.priority_p.value=int(self.priority_p.value) # convert priority parameter to integer type + elif self.max_opt==None: # check if max_opt is None + self.priority_p.value=int(self.priority_p.value) # convert priority parameter to integer type + else: + self.priority_p.value=-1 # assign -1 to priority parameter + if self.priority_flag==True: # check if the priority flag is True + self.opt_counter[p]=0 # assign zero to opt_counter at process number try: - if self.nn.accuracy!=None: - batch_acc=self.nn.accuracy(output,labels_batch) - except AttributeError: + opt_counter=self.nn.opt_counter[0] # get the neural network's optimization counter + opt_counter.scatter_update(tf.IndexedSlices(0,p)) # update opt_counter with zero at process number + self.nn.opt_counter[0]=opt_counter # assign opt_counter back to neural network's optimization counter + except Exception as e: + try: + if self.nn.attenuate!=None: # check if the neural network has an attenuate function + raise e + except Exception: + pass + output,batch_loss,param=self.opt(data_batch,labels_batch,p,lock,g_lock) # call the opt function to get output, batch loss and parameters + self.param[7]=param # assign param to kernel's parameters + if self.priority_flag==True: # check if the priority flag is True + opt_counter=np.frombuffer(self.opt_counter.get_obj(),dtype='i') # get a numpy array from opt_counter shared array object + opt_counter+=1 # increment opt_counter by one for each element + try: + opt_counter=self.nn.opt_counter[0] # get the neural network's optimization counter + opt_counter.assign(opt_counter+1) # increment opt_counter by one + self.nn.opt_counter[0]=opt_counter # assign opt_counter back to neural network's optimization counter + except Exception as e: + try: + if self.nn.attenuate!=None: # check if the neural network has an attenuate function + raise e + except Exception: + pass + try: + bc=self.nn.bc[0] # get the neural network's batch counter + bc.assign_add(1) # increment bc by one + self.nn.bc[0]=bc # assign bc back to neural network's batch counter + except Exception: pass try: - if self.nn.accuracy!=None: - self.total_loss[p]+=batch_loss - self.total_acc[p]+=batch_acc - except AttributeError: - self.total_loss[p]+=batch_loss - self.batch_counter[p]+=1 - if self.PO==1: - lock[1].acquire() - else: - lock[2].acquire() - batches=np.sum(self.batch_counter) - if batches>=self.batches: - batch_counter=np.frombuffer(self.batch_counter.get_obj(),dtype='i') - batch_counter*=0 - loss=np.sum(self.total_loss)/batches + batch_acc=self.nn.accuracy(output,labels_batch) # use the neural network's accuracy function to get the batch accuracy + except Exception as e: try: - if self.nn.accuracy!=None: - train_acc=np.sum(self.total_acc)/batches - except AttributeError: + if self.nn.accuracy!=None: # check if the neural network has an accuracy function + raise e + except Exception: + pass + try: + if self.nn.accuracy!=None: # check if the neural network has an accuracy function + self.total_loss[p]+=batch_loss # accumulate batch loss to total loss at process number + self.total_acc[p]+=batch_acc # accumulate batch accuracy to total accuracy at process number + except Exception: + self.total_loss[p]+=batch_loss # accumulate batch loss to total loss at process number + self.batch_counter[p]+=1 # increment batch counter at process number + if self.PO==1 or self.PO==2: # check if the optimizer order is 1 or 2 (lock before or after gradient calculation) + lock[1].acquire() # acquire the second lock + elif lock!=None: # check if lock is not None + lock.acquire() # acquire the lock + batches=np.sum(self.batch_counter) # sum up the batch counter for all processes + if batches>=self.batches: # check if the number of batches reaches the total number of batches + batch_counter=np.frombuffer(self.batch_counter.get_obj(),dtype='i') # get a numpy array from batch counter shared array object + batch_counter*=0 # reset batch counter to zero for each element + loss=np.sum(self.total_loss)/batches # calculate the average loss for all batches + try: + if self.nn.accuracy!=None: # check if the neural network has an accuracy function + train_acc=np.sum(self.total_acc)/batches # calculate the average accuracy for all batches + except Exception: pass - self.total_epoch.value+=1 - self.train_loss.value=loss - self.train_loss_list.append(loss) + self.total_epoch.value+=1 # increment total epoch by one + self.train_loss.value=loss # assign loss to train loss + self.train_loss_list.append(loss) # append loss to train loss list try: - if self.nn.accuracy!=None: - self.train_acc.value=train_acc - self.train_acc_list.append(train_acc) - except AttributeError: + if self.nn.accuracy!=None: # check if the neural network has an accuracy function + self.train_acc.value=train_acc # assign train_acc to train accuracy + self.train_acc_list.append(train_acc) # append train_acc to train accuracy list + except Exception: pass - if self.test_flag==True: - self.test_loss,self.test_acc=self.test(self.test_data,self.test_labels,test_batch,p) - self.test_loss_list.append(self.test_loss) + if self.test_flag==True: # check if the test flag is True + self.test_loss.value,self.test_acc.value=self.test(self.test_data,self.test_labels,test_batch,p) # call the test function to get test loss and accuracy + self.test_loss_list.append(self.test_loss.value) # append test loss to test loss list try: - if self.nn.accuracy!=None: - self.test_acc_list.append(self.test_acc) - except AttributeError: + if self.nn.accuracy!=None: # check if the neural network has an accuracy function + self.test_acc_list.append(self.test_acc.value) # append test accuracy to test accuracy list + except Exception: pass - self.print_save() - self.epoch_counter.value+=1 - total_loss=np.frombuffer(self.total_loss.get_obj(),dtype='f') - total_loss*=0 - if self.PO==1: - lock[1].release() - else: - lock[2].release() - if self.epoch_counter.value==self.epoch: - self.param[7]=param + self.print_save() # call the print_save function to print and save the results + self.epoch_counter.value+=1 # increment epoch counter by one + try: + ec=self.nn.ec[0] # get the neural network's epoch counter + ec.assign_add(1) # increment ec by one + self.nn.ec[0]=ec # assign ec back to neural network's epoch counter + except Exception: + pass + total_loss=np.frombuffer(self.total_loss.get_obj(),dtype='f') # get a numpy array from total loss shared array object + total_loss*=0 # reset total loss to zero for each element + try: + total_acc=np.frombuffer(self.total_acc.get_obj(),dtype='f') # get a numpy array from total accuracy shared array object + total_acc*=0 # reset total accuracy to zero for each element + except Exception as e: + try: + if self.nn.accuracy!=None: # check if the neural network has an accuracy function + raise e + except Exception: + pass + if self.PO==1 or self.PO==2: # check if the optimizer order is 1 or 2 (lock before or after gradient calculation) + lock[1].release() # release the second lock + elif lock!=None: # check if lock is not None + lock.release() # release the lock + if self.epoch_counter.value>=self.epoch: # check if the epoch counter reaches the epoch number + self.param[7]=param # assign param to kernel's parameters return - #Main training function. - def train(self,p,lock,test_batch=None): - self.train_counter+=1 - if self.epoch!=None: - if self.train_dataset!=None: - train_ds=self.train_dataset + + def train(self,p,lock=None,g_lock=None,test_batch=None): + if self.epoch!=None: # check if epoch is not None + if self.train_dataset!=None: # check if train dataset is not None + train_ds=self.train_dataset # assign train dataset to train_ds else: - if self.data_segment_flag==True: - train_ds=tf.data.Dataset.from_tensor_slices((self.train_data[p],self.train_labels[p])).batch(self.batch) - elif self.buffer_size!=None: - train_ds=tf.data.Dataset.from_tensor_slices((self.train_data,self.train_labels)).shuffle(self.buffer_size).batch(self.batch) + if self.data_segment_flag==True: # check if data segment flag is True + train_ds=tf.data.Dataset.from_tensor_slices((self.train_data[p],self.train_labels[p])).batch(self.batch) # create a dataset from tensor slices of segmented data and labels and batch them + elif self.buffer_size!=None: # check if buffer size is not None + train_ds=tf.data.Dataset.from_tensor_slices((self.train_data,self.train_labels)).shuffle(self.buffer_size).batch(self.batch) # create a dataset from tensor slices of data and labels and shuffle and batch them with buffer size else: - train_ds=tf.data.Dataset.from_tensor_slices((self.train_data,self.train_labels)).batch(self.batch) - self.train7(train_ds,p,test_batch,lock) + train_ds=tf.data.Dataset.from_tensor_slices((self.train_data,self.train_labels)).batch(self.batch) # create a dataset from tensor slices of data and labels and batch them + self.train7(train_ds,p,test_batch,lock,g_lock) # call the train7 function with train_ds, process number, test batch, lock and g_lock return def test(self,test_data=None,test_labels=None,batch=None,p=None): - if batch!=None: - total_loss=0 - total_acc=0 - if self.test_dataset!=None: - for data_batch,labels_batch in self.test_dataset: - try: - output=self.nn.fp(data_batch) - except TypeError: - output=self.nn.fp(data_batch,p) - batch_loss=self.nn.loss(output,labels_batch) - total_loss+=batch_loss + if type(self.nn.param[0])!=list: + test_data=test_data.astype(self.nn.param[0].dtype.name) # convert the test data type to match the model parameter type + test_labels=test_labels.astype(self.nn.param[0].dtype.name) # convert the test labels type to match the model parameter type + else: + test_data=test_data.astype(self.nn.param[0][0].dtype.name) # convert the test data type to match the model parameter type + test_labels=test_labels.astype(self.nn.param[0][0].dtype.name) # convert the test labels type to match the model parameter type + if batch!=None: # check if batch is not None + total_loss=0 # initialize total loss to zero + total_acc=0 # initialize total accuracy to zero + if self.test_dataset!=None: # check if test dataset is not None + for data_batch,labels_batch in self.test_dataset: # loop over the test dataset batches try: - if self.nn.accuracy!=None: - batch_acc=self.nn.accuracy(output,labels_batch) - total_acc+=batch_acc - except AttributeError: - pass - else: - total_loss=0 - total_acc=0 - batches=int((test_data.shape[0]-test_data.shape[0]%batch)/batch) - shape0=test_data.shape[0] - for j in range(batches): - index1=j*batch - index2=(j+1)*batch - data_batch=test_data[index1:index2] - labels_batch=test_labels[index1:index2] + try: + output=self.nn.fp(data_batch) # use the neural network's forward propagation function to get the output + except Exception: + output=self.nn.fp(data_batch,p) # use the neural network's forward propagation function to get the output with the process number + except Exception as e: + raise e + batch_loss=self.nn.loss(output,labels_batch) # use the neural network's loss function to get the batch loss + total_loss+=batch_loss # accumulate batch loss to total loss try: - output=self.nn.fp(data_batch) - except TypeError: - output=self.nn.fp(data_batch,p) - batch_loss=self.nn.loss(output,labels_batch) - total_loss+=batch_loss + batch_acc=self.nn.accuracy(output,labels_batch) # use the neural network's accuracy function to get the batch accuracy + total_acc+=batch_acc # accumulate batch accuracy to total accuracy + except Exception as e: + try: + if self.nn.accuracy!=None: # check if the neural network has an accuracy function + raise e + except Exception: + pass + else: # if test dataset is None + total_loss=0 # initialize total loss to zero + total_acc=0 # initialize total accuracy to zero + batches=int((test_data.shape[0]-test_data.shape[0]%batch)/batch) # calculate the number of batches + shape0=test_data.shape[0] # get the number of samples in the test data array + for j in range(batches): # loop over the batches + index1=j*batch # get the start index of the batch + index2=(j+1)*batch # get the end index of the batch + data_batch=test_data[index1:index2] # get the data batch from test data array + if type(test_labels)==list: # check if test labels is a list + for i in range(len(test_labels)): + labels_batch[i]=test_labels[i][index1:index2] # get the labels batch from test labels list + else: + labels_batch=test_labels[index1:index2] # get the labels batch from test labels array try: - if self.nn.accuracy!=None: - batch_acc=self.nn.accuracy(output,labels_batch) - total_acc+=batch_acc - except AttributeError: - pass - if shape0%batch!=0: - batches+=1 - index1=batches*batch - index2=batch-(shape0-batches*batch) + try: + output=self.nn.fp(data_batch) # use the neural network's forward propagation function to get the output + except Exception: + output=self.nn.fp(data_batch,p) # use the neural network's forward propagation function to get the output with the process number + except Exception as e: + raise e + batch_loss=self.nn.loss(output,labels_batch) # use the neural network's loss function to get the batch loss + total_loss+=batch_loss # accumulate batch loss to total loss try: - data_batch=tf.concat([test_data[index1:],test_data[:index2]],0) - labels_batch=tf.concat([test_labels[index1:],test_labels[:index2]],0) - except: - data_batch=tf.concat([test_data[index1:],test_data[:index2]],0) - labels_batch=tf.concat([test_labels[index1:],test_labels[:index2]],0) + batch_acc=self.nn.accuracy(output,labels_batch) # use the neural network's accuracy function to get the batch accuracy + total_acc+=batch_acc # accumulate batch accuracy to total accuracy + except Exception as e: + try: + if self.nn.accuracy!=None: # check if the neural network has an accuracy function + raise e + except Exception: + pass + if shape0%batch!=0: # check if there are remaining samples that are not in a batch + batches+=1 # increment batches by one + index1=batches*batch # get the start index of the remaining samples + index2=batch-(shape0-batches*batch) # get the number of samples that need to be filled with extra samples + data_batch=tf.concat([test_data[index1:],test_data[:index2]],0) # concatenate the remaining samples and extra samples to form a data batch + labels_batch=tf.concat([test_labels[index1:],test_labels[:index2]],0) # concatenate the remaining labels and extra labels to form a labels batch try: - output=self.nn.fp(data_batch) - except TypeError: - output=self.nn.fp(data_batch,p) - batch_loss=self.nn.loss(output,labels_batch) - total_loss+=batch_loss + try: + output=self.nn.fp(data_batch) # use the neural network's forward propagation function to get the output + except Exception: + output=self.nn.fp(data_batch,p) # use the neural network's forward propagation function to get the output with the process number + except Exception as e: + raise e + batch_loss=self.nn.loss(output,labels_batch) # use the neural network's loss function to get the batch loss + total_loss+=batch_loss # accumulate batch loss to total loss try: - if self.nn.accuracy!=None: - batch_acc=self.nn.accuracy(output,labels_batch) - total_acc+=batch_acc - except AttributeError: - pass - test_loss=total_loss.numpy()/batches + batch_acc=self.nn.accuracy(output,labels_batch) # use the neural network's accuracy function to get the batch accuracy + total_acc+=batch_acc # accumulate batch accuracy to total accuracy + except Exception as e: + try: + if self.nn.accuracy!=None: # check if the neural network has an accuracy function + raise e + except Exception: + pass + test_loss=total_loss.numpy()/batches # calculate the average test loss for all batches try: - if self.nn.accuracy!=None: - test_acc=total_acc.numpy()/batches - except AttributeError: + if self.nn.accuracy!=None: # check if the neural network has an accuracy function + test_acc=total_acc.numpy()/batches # calculate the average test accuracy for all batches + except Exception: pass - else: + else: # if batch is None try: - output=self.nn.fp(test_data) - except TypeError: - output=self.nn.fp(test_data,p) - test_loss=self.nn.loss(output,test_labels) - test_loss=test_loss.numpy() + try: + output=self.nn.fp(test_data) # use the neural network's forward propagation function to get the output + except Exception: + output=self.nn.fp(test_data,p) # use the neural network's forward propagation function to get the output with the process number + except Exception as e: + raise e + test_loss=self.nn.loss(output,test_labels) # use the neural network's loss function to get the test loss + test_loss=test_loss.numpy() # convert test loss to numpy array try: - if self.nn.accuracy!=None: - test_acc=self.nn.accuracy(output,test_labels) - test_acc=test_acc.numpy() - except AttributeError: - pass + test_acc=self.nn.accuracy(output,test_labels) # use the neural network's accuracy function to get the test accuracy + test_acc=test_acc.numpy() # convert test accuracy to numpy array + except Exception as e: + try: + if self.nn.accuracy!=None: # check if the neural network has an accuracy function + raise e + except Exception: + pass try: - if self.nn.accuracy!=None: - return test_loss,test_acc - except AttributeError: - return test_loss,None + if self.nn.accuracy!=None: # check if the neural network has an accuracy function + return test_loss,test_acc # return test loss and accuracy + except Exception: + return test_loss,None # return test loss and None def stop_func(self): - if self.end(): - self.save(self.total_epoch.value,True) - self.save_flag.value=True - self.stop_flag.value=True + if self.end(): # check if any of the end conditions is met + self.save(self.total_epoch.value,True) # save the model with total epoch and True flag + self.save_flag.value=True # set save flag to True + self.stop_flag.value=True # set stop flag to True return True return False - def stop_func_(self,lock): - if self.stop==True: - if self.stop_flag.value==True or self.stop_func(): - lock.release() + def stop_func_(self,lock=None): + if self.stop==True: # check if stop is True + if self.stop_flag.value==True or self.stop_func(): # check if the stop flag is True or the stop function returns True + if self.PO!=3: # check if the optimizer order is not 3 (no lock) + lock.release() # release the lock return True + return False - def print_save(self): - if self.epoch!=None: - if self.muti_p!=None: - muti_p=self.muti_p-1 - if self.epoch%10!=0: - p=self.epoch-self.epoch%muti_p - p=int(p/muti_p) - if p==0: - p=1 - else: - p=self.epoch/(muti_p+1) - p=int(p) - if p==0: - p=1 - if self.epoch_%p==0: - if self.test_flag==False: - try: - if self.nn.accuracy!=None: - print('epoch:{0} loss:{1:.6f}'.format(self.total_epoch.value,self.train_loss.value)) - if self.acc_flag=='%': - print('epoch:{0} accuracy:{1:.1f}'.format(self.total_epoch.value,self.train_acc.value*100)) - else: - print('epoch:{0} accuracy:{1:.6f}'.format(self.total_epoch.value,self.train_acc.value)) - print() - except AttributeError: - print('epoch:{0} loss:{1:.6f}'.format(self.total_epoch.value,self.train_loss.value)) - print() - else: - try: - if self.nn.accuracy!=None: - print('epoch:{0} loss:{1:.6f},test loss:{2:.6f}'.format(self.total_epoch.value,self.train_loss.value,self.test_loss.value)) - if self.acc_flag=='%': - print('epoch:{0} accuracy:{1:.1f},test accuracy:{2:.1f}'.format(self.total_epoch.value,self.train_acc.value*100,self.test_acc.value*100)) - else: - print('epoch:{0} accuracy:{1:.1f},test accuracy:{2:.1f}'.format(self.total_epoch.value,self.train_acc.value,self.test_acc.value)) - print() - except AttributeError: - print('epoch:{0} loss:{1:.6f},test loss:{2:.6f}'.format(self.total_epoch,self.train_loss.value,self.test_loss.value)) - print() - if self.muti_s!=None: - muti_s=self.muti_s-1 - if self.epoch%10!=0: - s=self.epoch-self.epoch%muti_s - s=int(s/muti_s) - if s==0: - s=1 - else: - s=self.epoch/(muti_s+1) - s=int(s) - if s==0: - s=1 - if self.muti_save!=None and self.epoch_%s==0: - if self.muti_save==1: - self.save(self.total_epoch.value) - else: - self.save(self.total_epoch.value,False) - self.epoch_+=1 - return - - - def train_info(self): - params=1 - total_params=0 - for i in range(len(self.nn.param)): - for j in range(len(self.nn.param[i].shape)): - params*=self.nn.param[i].shape[j] - total_params+=params - params=1 - print() - print('total params:{0}'.format(total_params)) - print() - print('batch:{0}'.format(self.batch)) - print() - print('epoch:{0}'.format(self.total_epoch.value)) - print() - try: - print('learning rate:{0}'.format(self.nn.lr)) - print() - except AttributeError: - pass - print() - print('-------------------------------------') - print() - print('train loss:{0:.6f}'.format(self.train_loss.value)) - if self.acc_flag=='%': - print('train acc:{0:.1f}'.format(self.train_acc.value*100)) - else: - print('train acc:{0:.6f}'.format(self.train_acc.value)) - return - - - def test_info(self): - print() - print('test loss:{0:.6f}'.format(self.test_loss.value)) - if self.acc_flag=='%': - print('test acc:{0:.1f}'.format(self.test_acc.value*100)) - else: - print('test acc:{0:.6f}'.format(self.test_acc.value)) - return - - - def info(self): - self.train_info() - if self.test_flag==True: - print() - print('-------------------------------------') - self.test_info() - return - - def visualize_train(self): - print() - plt.figure(1) - plt.plot(np.arange(self.total_epoch.value),self.train_loss_list) - plt.title('train loss') - plt.xlabel('epoch') - plt.ylabel('loss') - print('train loss:{0:.6f}'.format(self.train_loss.value)) - try: - if self.nn.accuracy!=None: - plt.figure(2) - plt.plot(np.arange(self.total_epoch.value),self.train_acc_list) - plt.title('train acc') - plt.xlabel('epoch') - plt.ylabel('acc') - if self.acc_flag=='%': - print('train acc:{0:.1f}'.format(self.train_acc.value*100)) - else: - print('train acc:{0:.6f}'.format(self.train_acc.value)) - except AttributeError: - pass - return - - - def visualize_test(self): - print() - plt.figure(1) - plt.plot(np.arange(self.total_epoch.value),self.test_loss_list) - plt.title('test loss') - plt.xlabel('epoch') - plt.ylabel('loss') - print('test loss:{0:.6f}'.format(self.test_loss.value)) - try: - if self.nn.accuracy!=None: - plt.figure(2) - plt.plot(np.arange(self.total_epoch.value),self.test_acc_list) - plt.title('test acc') - plt.xlabel('epoch') - plt.ylabel('acc') - if self.acc_flag=='%': - print('test acc:{0:.1f}'.format(self.test_acc.value*100)) - else: - print('test acc:{0:.6f}'.format(self.test_acc.value)) - except AttributeError: - pass - return - - - def comparison(self): - print() - plt.figure(1) - plt.plot(np.arange(self.total_epoch.value),self.train_loss_list,'b-',label='train loss') - if self.test_flag==True: - plt.plot(np.arange(self.total_epoch.value),self.test_loss_list,'r-',label='test loss') - plt.xlabel('epoch') - plt.ylabel('loss') - print('train loss:{0}'.format(self.train_loss.value)) - plt.legend() + print() # print a blank line + plt.figure(1) # create a new figure + plt.plot(np.arange(self.total_epoch.value),self.train_loss_list) # plot the train loss list against the total epoch + plt.title('train loss') # set the title of the figure + plt.xlabel('epoch') # set the x-axis label of the figure + plt.ylabel('loss') # set the y-axis label of the figure + print('train loss:{0:.6f}'.format(self.train_loss.value)) # print the train loss value with six decimal places try: - if self.nn.accuracy!=None: - plt.figure(2) - plt.plot(np.arange(self.total_epoch.value),self.train_acc_list,'b-',label='train acc') - if self.test_flag==True: - plt.plot(np.arange(self.total_epoch.value),self.test_acc_list,'r-',label='test acc') - plt.xlabel('epoch') - plt.ylabel('acc') - plt.legend() - if self.acc_flag=='%': - print('train acc:{0:.1f}'.format(self.train_acc.value*100)) + if self.nn.accuracy!=None: # check if the neural network has an accuracy function + plt.figure(2) # create a new figure + plt.plot(np.arange(self.total_epoch.value),self.train_acc_list) # plot the train accuracy list against the total epoch + plt.title('train acc') # set the title of the figure + plt.xlabel('epoch') # set the x-axis label of the figure + plt.ylabel('acc') # set the y-axis label of the figure + if self.acc_flag=='%': # check if the accuracy format is percentage + print('train acc:{0:.1f}'.format(self.train_acc.value*100)) # print the train accuracy value with one decimal place and percentage sign else: - print('train acc:{0:.6f}'.format(self.train_acc.value)) - except AttributeError: + print('train acc:{0:.6f}'.format(self.train_acc.value)) # print the train accuracy value with six decimal places + except Exception: pass - if self.test_flag==True: - print() - print('-------------------------------------') - print() - print('test loss:{0:.6f}'.format(self.test_loss.value)) - if self.acc_flag=='%': - print('test acc:{0:.1f}'.format(self.test_acc.value*100)) - else: - print('test acc:{0:.6f}'.format(self.test_acc.value)) - return - - - def save_p(self): - parameter_file=open('param.dat','wb') - pickle.dump(self.param[7],parameter_file) - parameter_file.close() - return - - - def save(self,i=None,one=True): - if self.save_flag==True: - return - if one==True: - output_file=open(self.filename,'wb') - else: - filename=self.filename.replace(self.filename[self.filename.find('.'):],'-{0}.dat'.format(i)) - output_file=open(filename,'wb') - self.file_list.append([filename]) - if len(self.file_list)>self.s+1: - os.remove(self.file_list[0][0]) - del self.file_list[0] - pickle.dump(self.nn,output_file) - pickle.dump(self.param[7],output_file) - pickle.dump(self.batch,output_file) - pickle.dump(self.end_loss,output_file) - pickle.dump(self.end_acc,output_file) - pickle.dump(self.end_test_loss,output_file) - pickle.dump(self.end_test_acc,output_file) - pickle.dump(self.acc_flag,output_file) - pickle.dump(self.file_list,output_file) - pickle.dump(self.train_counter,output_file) - pickle.dump(self.train_loss.value,output_file) - pickle.dump(self.train_acc.value,output_file) - pickle.dump(self.train_loss_list,output_file) - pickle.dump(self.train_acc_list,output_file) - pickle.dump(self.test_flag,output_file) - if self.test_flag==True: - pickle.dump(self.test_loss.value,output_file) - pickle.dump(self.test_acc.value,output_file) - pickle.dump(self.test_loss_list,output_file) - pickle.dump(self.test_acc_list,output_file) - pickle.dump(self.total_epoch.value,output_file) - output_file.close() - return - - - def restore(self,s_path): - input_file=open(s_path,'rb') - self.nn=pickle.load(input_file) - self.param[7]=pickle.load(input_file) - self.batch=pickle.load(input_file) - self.end_loss=pickle.load(input_file) - self.end_acc=pickle.load(input_file) - self.end_test_loss=pickle.load(input_file) - self.end_test_acc=pickle.load(input_file) - self.acc_flag=pickle.load(input_file) - self.file_list=pickle.load(input_file) - self.train_counter=pickle.load(input_file) - self.train_loss.value=pickle.load(input_file) - self.train_acc.value=pickle.load(input_file) - self.train_loss_list=pickle.load(input_file) - self.train_acc_list=pickle.load(input_file) - self.test_flag=pickle.load(input_file) - if self.test_flag==True: - self.test_loss.value=pickle.load(input_file) - self.test_acc.value=pickle.load(input_file) - self.test_loss_list=pickle.load(input_file) - self.test_acc_list=pickle.load(input_file) - self.total_epoch.value=pickle.load(input_file) - input_file.close() return From 1991587cb072aea7c14d1bdb07b410ad9f163b56 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Wed, 28 Jun 2023 00:09:43 +0800 Subject: [PATCH 006/337] Update kernel_reduced.py --- reduced kernel/DL/process/kernel_reduced.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/reduced kernel/DL/process/kernel_reduced.py b/reduced kernel/DL/process/kernel_reduced.py index 8ed97bfe..5e1e2ead 100644 --- a/reduced kernel/DL/process/kernel_reduced.py +++ b/reduced kernel/DL/process/kernel_reduced.py @@ -85,8 +85,6 @@ def data(self,train_data=None,train_labels=None,test_data=None,test_labels=None, def segment_data(self): if len(self.train_data)!=self.process: # check if the train data is already segmented - data=None - labels=None segments=int((len(self.train_data)-len(self.train_data)%self.process)/self.process) # calculate the number of samples for each segment for i in range(self.process): # loop over the processes index1=i*segments # get the start index of the segment From 1eae35a430d4fd29cc13fdba35298792cdd053db Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Wed, 28 Jun 2023 20:36:34 +0800 Subject: [PATCH 007/337] Update kernel_reduced.py --- reduced kernel/RL/kernel_reduced.py | 1208 ++++++++++++++------------- 1 file changed, 615 insertions(+), 593 deletions(-) diff --git a/reduced kernel/RL/kernel_reduced.py b/reduced kernel/RL/kernel_reduced.py index 498278d5..2db73a49 100644 --- a/reduced kernel/RL/kernel_reduced.py +++ b/reduced kernel/RL/kernel_reduced.py @@ -1,648 +1,670 @@ import tensorflow as tf -from tensorflow import data as tf_data +from tensorflow.python.ops import state_ops +from tensorflow.python.util import nest +from multiprocessing import Value,Array import numpy as np import matplotlib.pyplot as plt import statistics -#You can analyze kernel by example. -''' -multithreading example: -import kernel_reduced as k #import kernel -import DQN as d -import threading -dqn=d.DQN(4,128,2) #create neural network object -kernel=k.kernel(dqn,5) #start kernel,use 5 thread to train -kernel.action_count=2 -kernel.set_up(epsilon=0.01,pool_size=10000,batch=64,update_step=10) -kernel.PO=1 #use PO1 -kernel.multiprocessing_threading=threading -kernel.lock=[threading.Lock(),threading.Lock(),threading.Lock()] -class thread(threading.Thread): - def run(self): - kernel.train(100) -for _ in range(5): - _thread=thread() - _thread.start() -for _ in range(5): - _thread.join() -kernel.loss_list or kernel.loss #view training loss -kernel.visualize_train() -kernel.reward #view reward -kernel.visualize_reward() -''' class kernel: - def __init__(self,nn=None,process_thread=None,save_episode=False): - self.nn=nn - if process_thread!=None: - self.process_thread_num=np.arange(process_thread) - self.process_thread_num=list(self.process_thread_num) - self.reward=np.zeros(process_thread,dtype=np.float32) - self.loss=np.zeros(process_thread,dtype=np.float32) - self.sc=np.zeros(process_thread,dtype=np.float32) - self.opt_counter=np.zeros(process_thread,dtype=np.float32) - self.multiprocessing_threading=None - self.state_pool={} - self.action_pool={} - self.next_state_pool={} - self.reward_pool={} - self.done_pool={} - self.episode_set=[] - self.epsilon=None - self.episode_step=None - self.pool_size=None - self.batch=None - self.episode_=None - self.episode=0 - self.update_step=None - self.trial_count=None - self.process_thread=process_thread - self.process_thread_counter=0 - self.lock=None - self.pool_lock=[] - self.probability_list=[] - self.running_flag_list=[] - self.finish_list=[] - self.running_flag=np.array(0,dtype=np.int8) - self.PN=True - self.PO=None - self.max_episode_count=None - self.save_episode=save_episode - self.filename='save.dat' - self.reward_list=[] - self.loss_list=[] - self.total_episode=0 - self.total_time=0 - - - def action_vec(self): - self.action_one=np.ones(self.action_count,dtype=np.int8) - return + def __init__(self,nn=None,process=None): + self.nn=nn # the neural network model + if process!=None: # the number of processes + self.reward=np.zeros(process,dtype=np.float32) # the reward array for each process + self.sc=np.zeros(process,dtype=np.int32) # the step counter array for each process + self.state_pool={} # the dictionary to store the state pool for each process + self.action_pool={} # the dictionary to store the action pool for each process + self.next_state_pool={} # the dictionary to store the next state pool for each process + self.reward_pool={} # the dictionary to store the reward pool for each process + self.done_pool={} # the dictionary to store the done flag pool for each process + self.epsilon=None # the epsilon value for epsilon-greedy policy + self.episode_step=None # the maximum number of steps per episode + self.pool_size=None # the maximum size of the experience pool + self.batch=None # the batch size for training + self.episode=0 # the episode counter + self.update_step=None # the frequency of updating the target network parameters + self.trial_count=None # the number of trials to calculate the average reward + self.process=process # the number of processes + self.process_counter=0 # the counter of running processes + self.probability_list=[] # the list to store the probability distribution of running processes + self.running_flag_list=[] # the list to store the running flag of each process + self.finish_list=[] # the list to store the finished processes + self.running_flag=[] # the list to store the running flag of all processes (including 0) + self.PO=None # the optimization strategy (1: lock, 2: global lock, 3: no lock) + self.priority_flag=False # whether to use priority optimization or not + self.priority_p=0 # the priority process index for optimization + self.max_opt=None # the maximum number of optimization steps per process before switching priority + self.stop=False # whether to stop training or not + self.save_flag=False # whether to save the model or not + self.stop_flag=False # whether to stop all processes or not + self.reward_list=[] # the list to store the total reward per episode + self.loss_list=[] # the list to store the average loss per episode + self.total_episode=0 # the total number of episodes - def set_up(self,epsilon=None,episode_step=None,pool_size=None,batch=None,update_step=None,trial_count=None,criterion=None): - if epsilon!=None: - self.epsilon=np.ones(self.process_thread)*epsilon - if episode_step!=None: - self.episode_step=episode_step - if pool_size!=None: - self.pool_size=pool_size - if batch!=None: - self.batch=batch - if update_step!=None: - self.update_step=update_step - if trial_count!=None: - self.trial_count=trial_count - if criterion!=None: - self.criterion=criterion - if epsilon!=None: - self.action_vec() + def init(self,manager): + self.state_pool=manager.dict(self.state_pool) # use manager.dict to share state pool among processes + self.action_pool=manager.dict(self.action_pool) # use manager.dict to share action pool among processes + self.next_state_pool=manager.dict(self.next_state_pool) # use manager.dict to share next state pool among processes + self.reward_pool=manager.dict(self.reward_pool) # use manager.dict to share reward pool among processes + self.done_pool=manager.dict(self.done_pool) # use manager.dict to share done flag pool among processes + self.reward=Array('f',self.reward) # use Array to share reward array among processes + if type(self.nn.param[0])!=list: # the loss array for each process + self.loss=np.zeros(self.process,dtype=self.nn.param[0].dtype.name) + else: + self.loss=np.zeros(self.process,dtype=self.nn.param[0][0].dtype.name) + self.loss=Array('f',self.loss) # use Array to share loss array among processes + self.sc=Array('f',self.sc) # use Array to share step counter array among processes + self.process_counter=Value('i',self.process_counter) # use Value to share process counter among processes + self.probability_list=manager.list(self.probability_list) # use manager.list to share probability list among processes + self.running_flag_list=manager.list(self.running_flag_list) # use manager.list to share running flag list among processes + self.finish_list=manager.list(self.finish_list) # use manager.list to share finish list among processes + self.running_flag=manager.list([0]) # use manager.list to share running flag of all processes (including 0) + self.reward_list=manager.list(self.reward_list) # use manager.list to store reward list among processes + self.loss_list=manager.list(self.loss_list) # use manager.list to store loss list among processes + self.total_episode=Value('i',self.total_episode) # use Value to share total episode among processes + self.priority_p=Value('i',self.priority_p) # use Value to share priority process index among processes + if self.priority_flag==True: # if priority optimization is enabled + self.opt_counter=Array('i',np.zeros(self.process,dtype=np.int32)) # use Array to store optimization counter for each process + try: + if self.nn.attenuate!=None: # if attenuation function is defined + self.nn.opt_counter=manager.list([self.nn.opt_counter]) # use manager.list to share optimization counter for attenuation function among processes + except Exception: + pass + try: + self.nn.ec=manager.list(self.nn.ec) # use manager.list to share episode counter for neural network model among processes + except Exception: + pass + try: + self.nn.bc=manager.list(self.nn.bc) # use manager.list to share batch counter for neural network model among processes + except Exception: + pass + self.stop_flag=Value('b',0) # use Value to share stop flag among processes + self.save_flag=Value('b',0) # use Value to share save flag among processes + self.param=manager.dict() # use manager.dict to share parameters among processes return - def epsilon_greedy_policy(self,s,epsilon): - action_prob=self.action_one*epsilon/len(self.action_one) - best_a=np.argmax(self.nn.nn.fp(s)) - action_prob[best_a]+=1-epsilon - return action_prob + def action_vec(self): # a method to create a vector of ones with the same length as the action space + self.action_one=np.ones(self.action_count,dtype=np.int8) # create a vector of ones with int8 type + return - def pool(self,s,a,next_s,r,done,t,index): - if self.PN==True: - self.pool_lock[index].acquire() - try: - if type(self.state_pool[index])!=np.ndarray and self.state_pool[index]==None: - self.state_pool[index]=s - if type(a)==int: - a=np.array(a) - self.action_pool[index]=np.expand_dims(a,axis=0) - else: - self.action_pool[index]=a - self.next_state_pool[index]=np.expand_dims(next_s,axis=0) - self.reward_pool[index]=np.expand_dims(r,axis=0) - self.done_pool[index]=np.expand_dims(done,axis=0) - else: - try: - self.state_pool[index]=np.concatenate((self.state_pool[index],s),0) - if type(a)==int: - a=np.array(a,np.int64) - self.action_pool[index]=np.concatenate((self.action_pool[index],np.expand_dims(a,axis=0)),0) - else: - self.action_pool[index]=np.concatenate((self.action_pool[index],a),0) - self.next_state_pool[index]=np.concatenate((self.next_state_pool[index],np.expand_dims(next_s,axis=0)),0) - self.reward_pool[index]=np.concatenate((self.reward_pool[index],np.expand_dims(r,axis=0)),0) - self.done_pool[index]=np.concatenate((self.done_pool[index],np.expand_dims(done,axis=0)),0) - except: - pass - if type(self.state_pool[index])==np.ndarray and len(self.state_pool[index])>self.pool_size: - self.state_pool[index]=self.state_pool[index][1:] - self.action_pool[index]=self.action_pool[index][1:] - self.next_state_pool[index]=self.next_state_pool[index][1:] - self.reward_pool[index]=self.reward_pool[index][1:] - self.done_pool[index]=self.done_pool[index][1:] - except: - self.pool_lock[index].release() - return - self.pool_lock[index].release() - else: - if type(self.state_pool[t])==np.ndarray and self.state_pool[t]==None: - self.state_pool[t]=s - if type(a)==int: - a=np.array(a) - self.action_pool[t]=np.expand_dims(a,axis=0) - else: - self.action_pool[t]=a - self.next_state_pool[t]=np.expand_dims(next_s,axis=0) - self.reward_pool[t]=np.expand_dims(r,axis=0) - self.done_pool[t]=np.expand_dims(done,axis=0) - else: - self.state_pool[t]=np.concatenate((self.state_pool[t],s),0) - if type(a)==int: - a=np.array(a,np.int64) - self.action_pool[t]=np.concatenate((self.action_pool[t],np.expand_dims(a,axis=0)),0) - else: - self.action_pool[t]=np.concatenate((self.action_pool[t],a),0) - self.next_state_pool[t]=np.concatenate((self.next_state_pool[t],np.expand_dims(next_s,axis=0)),0) - self.reward_pool[t]=np.concatenate((self.reward_pool[t],np.expand_dims(r,axis=0)),0) - self.done_pool[t]=np.concatenate((self.done_pool[t],np.expand_dims(done,axis=0)),0) - if type(self.state_pool[t])==np.ndarray and len(self.state_pool[t])>self.pool_size: - self.state_pool[t]=self.state_pool[t][1:] - self.action_pool[t]=self.action_pool[t][1:] - self.next_state_pool[t]=self.next_state_pool[t][1:] - self.reward_pool[t]=self.reward_pool[t][1:] - self.done_pool[t]=self.done_pool[t][1:] + def set_up(self,epsilon=None,episode_step=None,pool_size=None,batch=None,update_step=None,trial_count=None,criterion=None): # a method to set up some parameters for the kernel class + if epsilon!=None: # if epsilon value is given + self.epsilon=np.ones(self.process)*epsilon # create an array of epsilon values with the same length as the number of processes + if episode_step!=None: # if episode step value is given + self.episode_step=episode_step # assign the episode step value to the attribute + if pool_size!=None: # if pool size value is given + self.pool_size=pool_size # assign the pool size value to the attribute + if batch!=None: # if batch size value is given + self.batch=batch # assign the batch size value to the attribute + if update_step!=None: # if update step value is given + self.update_step=update_step # assign the update step value to the attribute + if trial_count!=None: # if trial count value is given + self.trial_count=trial_count # assign the trial count value to the attribute + if criterion!=None: # if criterion value is given + self.criterion=criterion # assign the criterion value to the attribute + if epsilon!=None: # if epsilon value is given + self.action_vec() # call the action_vec method to create a vector of ones with the same length as the action space return - def get_index(self,t): - if self.PN==True: - while len(self.running_flag_list)self.process_thread_counter: - self.running_flag_list[t]=self.running_flag[1:].copy() - while len(self.probability_list)self.pool_size: # if the state pool is a numpy array and its length exceeds the pool size + self.state_pool[index]=self.state_pool[index][1:] # remove the oldest state from the state pool + self.action_pool[index]=self.action_pool[index][1:] # remove the oldest action from the action pool + self.next_state_pool[index]=self.next_state_pool[index][1:] # remove the oldest next state from the next state pool + self.reward_pool[index]=self.reward_pool[index][1:] # remove the oldest reward from the reward pool + self.done_pool[index]=self.done_pool[index][1:] # remove the oldest done flag from the done flag pool + except Exception: # if any exception occurs + pool_lock[index].release() # release the lock for the process index + return # return from the method + pool_lock[index].release() # release the lock for the process index + return # return from the method - def end(self): - if self.trial_count!=None: - if len(self.reward_list)>=self.trial_count: - avg_reward=statistics.mean(self.reward_list[-self.trial_count:]) - if self.criterion!=None and avg_reward>=self.criterion: - return True + def get_index(self,p,lock): # a method to get a random process index according to the probability distribution of running processes + while len(self.running_flag_list)self.process_counter.value: # if the lengtn of the running flag list for the current process index is less tnan the process counter value or the sum of the running flag list for the current process index is greater tnan the process counter value + self.running_flag_list[p]=self.running_flag[1:].copy() # assign a copy of the running flag of all processes (excluding 0) to the running flag list for the current process index + while len(self.probability_list)=self.trial_count: # if the length of the reward list is greater than or equal to the trial count value + avg_reward=statistics.mean(self.reward_list[-self.trial_count:]) # calculate the average reward of the last trial count episodes + if self.criterion!=None and avg_reward>=self.criterion: # if criterion value is given and average reward is greater than or equal to criterion value + return True # return True to indicate termination condition is met + return False # return False to indicate termination condition is not met + + + @tf.function + def opt(self,state_batch,action_batch,next_state_batch,reward_batch,done_batch,p,lock,g_lock=None): # a method to optimize the neural network model for a given batch of experience data, process index and locks + with tf.GradientTape(persistent=True) as tape: # create a persistent gradient tape to record operations on tensors + try: try: - if self.nn.nn!=None: - gradient=tape.gradient(loss,self.nn.param) - try: - if self.nn.attenuate!=None: - gradient=self.nn.attenuate(gradient,self.opt_counter,t) - except AttributeError: - pass - except AttributeError: - actor_gradient=tape.gradient(loss[0],self.nn.param[0]) - critic_gradient=tape.gradient(loss[1],self.nn.param[1]) - try: - if self.nn.attenuate!=None: - actor_gradient=self.nn.attenuate(actor_gradient,self.opt_counter,t) - critic_gradient=self.nn.attenuate(critic_gradient,self.opt_counter,t) - except AttributeError: - pass + loss=self.nn.loss(state_batch,action_batch,next_state_batch,reward_batch,done_batch) # calculate the loss by using the loss method of the neural network model + except Exception: # if any exception occurs + loss=self.nn.loss(state_batch,action_batch,next_state_batch,reward_batch,done_batch,p) # calculate the loss by using the loss method of the neural network model witn the process index + except Exception as e: # if any exception occurs + raise e # raise the exception again + if self.PO==1: # if the optimization strategy is lock + if self.priority_flag==True and self.priority_p.value!=-1: # if priority optimization is enabled and priority process index is not -1 + while True: # loop until the current process index is equal to the priority process index + if p==self.priority_p.value: # if the current process index is equal to the priority process index + break # break the loop and continue optimization + else: # if the current process index is not equal to the priority process index + continue # skip optimization and continue looping + lock[0].acquire() # acquire the lock for optimization + if self.stop_func_(lock[0]): # call the stop_func_ method witn the lock for optimization and check if it returns True + return 0 # return from the method witn zero value try: - if self.nn.gradient!=None: - try: - self.nn.opt.apply_gradients(zip(gradient,self.nn.param)) - except AttributeError: - try: - self.nn.opt(gradient) - except TypeError: - self.nn.opt(gradient,t) - except AttributeError: try: - if self.nn.nn!=None: - self.nn.opt.apply_gradients(zip(gradient,self.nn.param)) - except AttributeError: - self.nn.opt.apply_gradients(zip(actor_gradient,self.nn.param[0])) - self.nn.opt.apply_gradients(zip(critic_gradient,self.nn.param[1])) + gradient=self.nn.gradient(tape,loss) # calculate the gradient by using the gradient method of the neural network model + except Exception: # if any exception occurs + try: + if self.nn.nn!=None: # if there is a neural network attribute in the neural network model + gradient=tape.gradient(loss,self.nn.param) # calculate the gradient by using the gradient method of the gradient tape with the loss and the neural network parameter + except Exception: # if any exception occurs + actor_gradient=tape.gradient(loss[0],self.nn.param[0]) # calculate the actor gradient by using the gradient method of the gradient tape with the first element of the loss and the first element of the neural network parameter + critic_gradient=tape.gradient(loss[1],self.nn.param[1]) # calculate the critic gradient by using the gradient method of the gradient tape with the second element of the loss and the second element of the neural network parameter + except Exception as e: # if any exception occurs + raise e # raise the exception again try: - if self.nn.attenuate!=None: - self.opt_counter+=1 - except AttributeError: - pass - self.lock[0].release() - elif self.PO==2: - self.lock[0].acquire() + try: + gradient=self.nn.attenuate(gradient,self.nn.opt_counter,p) # attenuate the gradient by using the attenuate method of the neural network model with the gradient, the optimization counter and the process index + except Exception: # if any exception occurs + actor_gradient=self.nn.attenuate(actor_gradient,self.nn.opt_counter,p) # attenuate the actor gradient by using the attenuate method of the neural network model with the actor gradient, the optimization counter and the process index + critic_gradient=self.nn.attenuate(critic_gradient,self.nn.opt_counter,p) # attenuate the critic gradient by using the attenuate method of the neural network model with the critic gradient, the optimization counter and the process index + except Exception as e: # if any exception occurs + try: + if self.nn.attenuate!=None: # if attenuation function is defined + raise e # raise the exception again + except Exception: # if attenuation function is not defined + pass # ignore it and pass try: - gradient=self.nn.gradient(tape,loss) - except AttributeError: try: - if self.nn.nn!=None: - gradient=tape.gradient(loss,self.nn.param) - except AttributeError: - actor_gradient=tape.gradient(loss[0],self.nn.param[0]) - critic_gradient=tape.gradient(loss[1],self.nn.param[1]) - self.lock[0].release() - self.lock[1].acquire() + param=self.nn.opt(gradient) # optimize the neural network parameter by using the opt method of the neural network model with the gradient + except Exception: # if any exception occurs + param=self.nn.opt(gradient,p) # optimize the neural network parameter by using the opt method of the neural network model witn the gradient and the process index + except Exception as e: # if any exception occurs + raise e # raise the exception again + lock[0].release() # release the lock for optimization + elif self.PO==2: # if the optimization strategy is global lock + g_lock.acquire() # acquire the global lock for calculating gradient + if self.stop_func_(g_lock): # call the stop_func_ method witn the global lock and check if it returns True + return 0 # return from the method witn zero value try: - if self.nn.attenuate!=None: + try: + gradient=self.nn.gradient(tape,loss) # calculate the gradient by using the gradient method of the neural network model + except Exception: # if any exception occurs try: - gradient=self.nn.attenuate(gradient,self.opt_counter,t) - except NameError: - actor_gradient=self.nn.attenuate(actor_gradient,self.opt_counter,t) - critic_gradient=self.nn.attenuate(critic_gradient,self.opt_counter,t) - except AttributeError: - pass + if self.nn.nn!=None: # if tnere is a neural network attribute in the neural network model + gradient=tape.gradient(loss,self.nn.param) # calculate the gradient by using the gradient method of the gradient tape witn the loss and the neural network parameter + except Exception: # if any exception occurs + actor_gradient=tape.gradient(loss[0],self.nn.param[0]) # calculate the actor gradient by using the gradient method of the gradient tape witn the first element of the loss and the first element of the neural network parameter + critic_gradient=tape.gradient(loss[1],self.nn.param[1]) # calculate the critic gradient by using the gradient method of the gradient tape witn the second element of the loss and the second element of the neural network parameter + except Exception as e: # if any exception occurs + raise e # raise the exception again + g_lock.release() # release the global lock for calculating gradient + if self.priority_flag==True and self.priority_p.value!=-1: # if priority optimization is enabled and priority process index is not -1 + while True: # loop until tn while True: # loop until the current process index is equal to the priority process index + if p==self.priority_p.value: # if the current process index is equal to the priority process index + break # break the loop and continue optimization + else: # if the current process index is not equal to the priority process index + continue # skip optimization and continue looping + lock[0].acquire() # acquire the lock for optimization + if self.stop_func_(lock[0]): # call the stop_func_ method with the lock for optimization and check if it returns True + return 0 # return from the method with zero value + try: + try: + gradient=self.nn.attenuate(gradient,self.nn.opt_counter,p) # attenuate the gradient by using the attenuate method of the neural network model with the gradient, the optimization counter and the process index + except Exception: # if any exception occurs + actor_gradient=self.nn.attenuate(actor_gradient,self.nn.opt_counter,p) # attenuate the actor gradient by using the attenuate method of the neural network model with the actor gradient, the optimization counter and the process index + critic_gradient=self.nn.attenuate(critic_gradient,self.nn.opt_counter,p) # attenuate the critic gradient by using the attenuate method of the neural network model with the critic gradient, the optimization counter and the process index + except Exception as e: # if any exception occurs + try: + if self.nn.attenuate!=None: # if attenuation function is defined + raise e # raise the exception again + except Exception: # if attenuation function is not defined + pass # ignore it and pass try: - if self.nn.gradient!=None: - try: - self.nn.opt.apply_gradients(zip(gradient,self.nn.param)) - except AttributeError: - try: - self.nn.opt(gradient) - except TypeError: - self.nn.opt(gradient,t) - except AttributeError: try: - if self.nn.nn!=None: - self.nn.opt.apply_gradients(zip(gradient,self.nn.param)) - except AttributeError: - self.nn.opt.apply_gradients(zip(actor_gradient,self.nn.param[0])) - self.nn.opt.apply_gradients(zip(critic_gradient,self.nn.param[1])) + param=self.nn.opt(gradient) # optimize the neural network parameter by using the opt method of the neural network model with the gradient + except Exception: # if any exception occurs + param=self.nn.opt(gradient,p) # optimize the neural network parameter by using the opt method of the neural network model witn the gradient and the process index + except Exception as e: # if any exception occurs + raise e # raise the exception again + lock[0].release() # release the lock for optimization + elif self.PO==3: # if the optimization strategy is no lock + if self.priority_flag==True and self.priority_p.value!=-1: # if priority optimization is enabled and priority process index is not -1 + while True: # loop until the current process index is equal to the priority process index + if p==self.priority_p.value: # if the current process index is equal to the priority process index + break # break the loop and continue optimization + else: # if the current process index is not equal to the priority process index + continue # skip optimization and continue looping + if self.stop_func_(): # call the stop_func_ method witnout any lock and check if it returns True + return 0 # return from the method witn zero value try: - if self.nn.attenuate!=None: - self.opt_counter+=1 - except AttributeError: - pass - self.lock[1].release() - return loss - - - def _train(self,t,j=None,batches=None,length=None,ln=None): - if j==batches-1: - index1=batches*self.batch - index2=self.batch-(length-batches*self.batch) - state_batch=np.concatenate((self.state_pool[t][index1:length],self.state_pool[t][:index2]),0) - action_batch=np.concatenate((self.action_pool[t][index1:length],self.action_pool[t][:index2]),0) - next_state_batch=np.concatenate((self.next_state_pool[t][index1:length],self.next_state_pool[t][:index2]),0) - reward_batch=np.concatenate((self.reward_pool[t][index1:length],self.reward_pool[t][:index2]),0) - done_batch=np.concatenate((self.done_pool[t][index1:length],self.done_pool[t][:index2]),0) - loss=self.opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch,t) - self.loss[t]+=loss + try: + gradient=self.nn.gradient(tape,loss) # calculate the gradient by using the gradient method of the neural network model + except Exception: # if any exception occurs + try: + if self.nn.nn!=None: # if tnere is a neural network attribute in the neural network model + gradient=tape.gradient(loss,self.nn.param) # calculate the gradient by using the gradient method of the gradient tape witn the loss and the neural network parameter + except Exception: # if any exception occurs + actor_gradient=tape.gradient(loss[0],self.nn.param[0]) # calculate the actor gradient by using the gradient method of the gradient tape witn the first element of the loss and the first element of the neural network parameter + critic_gradient=tape.gradient(loss[1],self.nn.param[1]) # calculate the critic gradient by using the gradient method of the gradient tape witn the second element of the loss and the second element of the neural network parameter + except Exception as e: # if any exception occurs + raise e # raise the exception again try: - self.nn.bc[t]+=1 - except AttributeError: - pass - else: - index1=j*self.batch - index2=(j+1)*self.batch - state_batch=self.state_pool[t][index1:index2] - action_batch=self.action_pool[t][index1:index2] - next_state_batch=self.next_state_pool[t][index1:index2] - reward_batch=self.reward_pool[t][index1:index2] - done_batch=self.done_pool[t][index1:index2] - loss=self.opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch,t) - self.loss[t]+=loss + try: + gradient=self.nn.attenuate(gradient,self.nn.opt_counter,p) # attenuate the gradient by using the attenuate method of the neural network model with the gradient, the optimization counter and the process index + except Exception: # if any exception occurs + actor_gradient=self.nn.attenuate(actor_gradient,self.nn.opt_counter,p) # attenuate the actor gradient by using the attenuate method of the neural network model with the actor gradient, the optimization counter and the process index + critic_gradient=self.nn.attenuate(critic_gradient,self.nn.opt_counter,p) # attenuate the critic gradient by using the attenuate method of the neural network model with the critic gradient, the optimization counter and the process index + except Exception as e: # if any exception occurs + try: + if self.nn.attenuate!=None: # if attenuation function is defined + raise e # raise the exception again + except Exception: # if attenuation function is not defined + pass # ignore it and pass try: - self.nn.bc[t]=j - except AttributeError: - pass - return + try: + param=self.nn.opt(gradient) # optimize the neural network parameter by using the opt method of the neural network model with the gradient + except Exception: # if any exception occurs + param=self.nn.opt(gradient,p) # optimize the neural network parameter by using the opt method of the neural network model witn the gradient and the process index + except Exception as e: # if any exception occurs + raise e # raise the exception again + return loss,param # return the loss and the parameter from the method + + def update_nn_param(self,param=None): # a method to update the neural network parameter witn a given parameter or the attribute parameter + if param==None: # if no parameter is given + parameter_flat=nest.flatten(self.nn.param) # flatten the neural network parameter to a list of tensors + parameter7_flat=nest.flatten(self.param[7]) # flatten the attribute parameter to a list of tensors + else: # if a parameter is given + parameter_flat=nest.flatten(self.nn.param) # flatten the neural network parameter to a list of tensors + parameter7_flat=nest.flatten(param) # flatten the given parameter to a list of tensors + for i in range(len(parameter_flat)): # loop tnrougn the lengtn of the flattened parameters + if param==None: # if no parameter is given + state_ops.assign(parameter_flat[i],parameter7_flat[i]) # assign the value of the attribute parameter to the neural network parameter + else: # if a parameter is given + state_ops.assign(parameter_flat[i],parameter7_flat[i]) # assign the value of the given parameter to the neural network parameter + self.nn.param=nest.pack_sequence_as(self.nn.param,parameter_flat) # pack the flattened list of tensors back to the original structure of the neural network parameter + self.param[7]=nest.pack_sequence_as(self.param[7],parameter7_flat) # pack the flattened list of tensors back to the original structure of the attribute parameter + return # return from the method - def train_(self,t,ln=None): - train_ds=tf_data.Dataset.from_tensor_slices((self.state_pool[t],self.action_pool[t],self.next_state_pool[t],self.reward_pool[t],self.done_pool[t])).shuffle(len(self.state_pool[t])).batch(self.batch) - for state_batch,action_batch,next_state_batch,reward_batch,done_batch in train_ds: - self.suspend_func(t) - loss=self.opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch,t) - self.loss[t]+=loss + + def _train(self,p,j,batches,length,lock,g_lock): # a method to train the neural network model for a given process index, batch index, number of batches, pool lengtn and locks + if j==batches-1: # if it is the last batch + index1=batches*self.batch # get the start index of the last batch by multiplying batches and batch size + index2=self.batch-(length-batches*self.batch) # get tn index2=self.batch-(length-batches*self.batch) # get the end index of the last batch by subtracting the pool length from the product of batches and batch size and then subtracting the result from the batch size + state_batch=np.concatenate((self.state_pool[p][index1:length],self.state_pool[p][:index2]),0) # concatenate the state pool from the start index to the pool length and the state pool from zero to the end index along axis 0 to get the state batch + action_batch=np.concatenate((self.action_pool[p][index1:length],self.action_pool[p][:index2]),0) # concatenate the action pool from the start index to the pool length and the action pool from zero to the end index along axis 0 to get the action batch + next_state_batch=np.concatenate((self.next_state_pool[p][index1:length],self.next_state_pool[p][:index2]),0) # concatenate the next state pool from the start index to the pool length and the next state pool from zero to the end index along axis 0 to get the next state batch + reward_batch=np.concatenate((self.reward_pool[p][index1:length],self.reward_pool[p][:index2]),0) # concatenate the reward pool from the start index to the pool length and the reward pool from zero to the end index along axis 0 to get the reward batch + done_batch=np.concatenate((self.done_pool[p][index1:length],self.done_pool[p][:index2]),0) # concatenate the done flag pool from the start index to the pool length and the done flag pool from zero to the end index along axis 0 to get the done flag batch + if self.PO==2: # if the optimization strategy is global lock + if type(g_lock)!=list: # if g_lock is not a list + pass # do nothing and pass + elif len(g_lock)==self.process: # if g_lock is a list with the same length as the number of processes + ln=p # assign p (the current process index) to ln (the lock number) + g_lock=g_lock[ln] # assign g_lock[ln] (the lock for ln) to g_lock + else: # if g_lock is a list with a different length than the number of processes + ln=int(np.random.choice(len(g_lock))) # sample a random integer from zero to the length of g_lock as ln (the lock number) + g_lock=g_lock[ln] # assign g_lock[ln] (the lock for ln) to g_lock + loss,param=self.opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch,p,lock,g_lock) # call the opt method witn the batcn data, the process index, the lock for optimization and the global lock for calculating gradient and get the loss and the parameter + else: # if the optimization strategy is not global lock + loss,param=self.opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch,p,lock) # call the opt method witn the batcn data, the process index and the lock for optimization and get the loss and the parameter + self.param[7]=param # assign the parameter to the attribute parameter + self.loss[p]+=loss # add the loss to the loss array for the process index try: - self.nn.bc[t]+=1 - except AttributeError: - pass - return - + bc=self.nn.bc[0] # get the batcn counter for neural network model + bc.assign_add(1) # increment the batcn counter by one + self.nn.bc[0]=bc # assign the updated batcn counter back to neural network model + except Exception: # if any exception occurs + pass # ignore it and pass + else: # if it is not the last batcn + index1=j*self.batch # get the start index of the current batcn by multiplying j (the batcn index) and batcn size + index2=(j+1)*self.batch # get tn index2=(j+1)*self.batch # get the end index of the current batch by adding one to j (the batch index) and multiplying by batch size + state_batch=self.state_pool[p][index1:index2] # get the state batch by slicing the state pool from the start index to the end index + action_batch=self.action_pool[p][index1:index2] # get the action batch by slicing the action pool from the start index to the end index + next_state_batch=self.next_state_pool[p][index1:index2] # get the next state batch by slicing the next state pool from the start index to the end index + reward_batch=self.reward_pool[p][index1:index2] # get the reward batch by slicing the reward pool from the start index to the end index + done_batch=self.done_pool[p][index1:index2] # get the done flag batch by slicing the done flag pool from the start index to the end index + if self.PO==2: # if the optimization strategy is global lock + if type(g_lock)!=list: # if g_lock is not a list + pass # do nothing and pass + elif len(g_lock)==self.process: # if g_lock is a list with the same length as the number of processes + ln=p # assign p (the current process index) to ln (the lock number) + g_lock=g_lock[ln] # assign g_lock[ln] (the lock for ln) to g_lock + else: # if g_lock is a list with a different length than the number of processes + ln=int(np.random.choice(len(g_lock))) # sample a random integer from zero to the length of g_lock as ln (the lock number) + g_lock=g_lock[ln] # assign g_lock[ln] (the lock for ln) to g_lock + loss,param=self.opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch,p,lock,g_lock) # call the opt method witn the batcn data, the process index, the lock for optimization and the global lock for calculating gradient and get the loss and the parameter + else: # if the optimization strategy is not global lock + loss,param=self.opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch,p,lock) # call the opt method witn the batcn data, the process index and the lock for optimization and get the loss and the parameter + self.param[7]=param # assign the parameter to the attribute parameter + self.loss[p]+=loss # add the loss to the loss array for the process index + try: + bc=self.nn.bc[0] # get the batcn counter for neural network model + bc.assign_add(1) # increment the batcn counter by one + self.nn.bc[0]=bc # assign the updated batcn counter back to neural network model + except Exception: # if any exception occurs + pass # ignore it and pass + return # return from the method - def _train_(self,t): - if len(self.done_pool[t])=self.max_opt: # if max_opt value is given and the optimization counter of the priority process index is greater than or equal to max_opt value + self.priority_p.value=int(self.priority_p.value) # convert the priority process index to an integer + elif self.max_opt==None: # if max_opt value is not given + self.priority_p.value=int(self.priority_p.value) # convert the priority process index to an integer + else: # if max_opt value is given and the optimization counter of the priority process index is less than max_opt value + self.priority_p.value=-1 # assign -1 to the priority process index to indicate no priority + if self.priority_flag==True: # if priority optimization is enabled + self.opt_counter[p]=0 # reset the optimization counter for the current process index to zero + try: + if self.nn.attenuate!=None: # if attenuation function is defined + opt_counter=self.nn.opt_counter[0] # get the optimization counter for attenuation function + opt_counter.scatter_update(tf.IndexedSlices(0,p)) # update the optimization counter for attenuation function by setting the value at the current process index to zero + self.nn.opt_counter[0]=opt_counter # assign the updated optimization counter back to attenuation function + except Exception: # if any exception occurs + pass # ignore it and pass + self._train(p,j,batches,length,lock,g_lock) # call the _train method with the current process index, batch index, number of batches, pool length and locks + if self.priority_flag==True: # if priority optimization is enabled + opt_counter=np.frombuffer(self.opt_counter.get_obj(),dtype='i') # get the optimization counter array from the shared memory buffer with int32 type + opt_counter+=1 # increment the optimization counter array by one try: - self.nn.bc[t]=0 - except AttributeError: - pass - self.train_(t) - if self.PN==True: - if self.PO==1 or self.PO==3: - self.lock[1].acquire() - else: - self.lock[2].acquire() - else: - self.lock[1].acquire() - if self.update_step!=None: - if self.sc[t]%self.update_step==0: - self.nn.update_param() - else: - self.nn.update_param() - if self.PN==True: - if self.PO==1 or self.PO==3: - self.lock[1].release() - else: - self.lock[2].release() - else: - self.lock[1].release() - self.loss[t]=self.loss[t]/batches - self.sc[t]+=1 + if self.nn.attenuate!=None: # if attenuation function is defined + opt_counter=self.nn.opt_counter[0] # get the optimization counter for attenuation function + opt_counter.assign(opt_counter+1) # increment the optimization counter for attenuation function by one + self.nn.opt_counter[0]=opt_counter # assign the updated optimization counter back to attenuation function + except Exception: # if any exception occurs + pass # ignore it and pass + if self.PO==1 or self.PO==2: # if the optimization strategy is lock or global lock + lock[1].acquire() # acquire the lock for updating target network parameters + if self.update_step!=None: # if update step value is given + if self.sc[p]%self.update_step==0: # if the step counter array for the process index is divisible by update step value + self.nn.update_param() # call the update_param method of the neural network model to update the target network parameters + else: # if update step value is not given + self.nn.update_param() # call the update_param method of the neural network model to update the target network parameters + if self.PO==1 or self.PO==2: # if the optimization strategy is lock or global lock + lock[1].release() # release the lock for updating target network parameters + self.loss[p]=self.loss[p]/batches # calculate the average loss for the process index by dividing the loss array by the number of batcnes + self.sc[p]+=1 # increment the step counter array for the process index by one try: - self.nn.ec[t]+=1 - except AttributeError: - pass - return + ec=self.nn.ec[0] # get the episode counter for neural network model + ec.assign_add(1) # increment the episode counter by one + self.nn.ec[0]=ec # assign the updated episode counter back to neural network model + except Exception: # if any exception occurs + pass # ignore it and pass + return # return from the method - def train(self,episode_count): - if self.PN==True: - if self.PO==1 or self.PO==3: - self.lock[2].acquire() - else: - self.lock[3].acquire() - else: - self.lock[0].acquire() - t=self.process_thread_num.pop(0) - t=int(t) - self.state_pool[t]=None - self.action_pool[t]=None - self.next_state_pool[t]=None - self.reward_pool[t]=None - self.done_pool[t]=None - self.running_flag=np.append(self.running_flag,np.array(1,dtype=np.int8)) - if self.multiprocessing_threading!=None: - self.pool_lock.append(self.multiprocessing_threading.Lock()) - self.process_thread_counter+=1 - self.finish_list.append(None) - try: - epsilon=self.epsilon[t] - except: - epsilon=None + def train(self,p,episode_count,lock,pool_lock,g_lock=None): # a method to execute a certain number of training episodes for a given process index and locks + if self.PO==1 or self.PO==2: # if the optimization strategy is lock or global lock + lock[1].acquire() # acquire the lock for initializing the pools and flags + elif self.PO==3: # if the optimization strategy is no lock + lock[1].acquire() # acquire the lock for initializing the pools and flags + self.state_pool[p]=None # initialize the state pool for the process index with None value + self.action_pool[p]=None # initialize the action pool for the process index with None value + self.next_state_pool[p]=None # initialize the next state pool for the process index with None value + self.reward_pool[p]=None # initialize the reward pool for the process index with None value + self.done_pool[p]=None # initialize the done flag pool for the process index with None value + self.running_flag.append(1) # append a one value to the running flag list to indicate the process is running + self.process_counter.value+=1 # increment the process counter by one + self.finish_list.append(None) # append a None value to the finish list to indicate the process is not finished try: - self.nn.ec.append(0) - except AttributeError: - pass - try: - self.nn.bc.append(0) - except AttributeError: - pass - if self.PN==True: - if self.PO==1 or self.PO==3: - self.lock[2].release() - else: - self.lock[3].release() - else: - self.lock[0].release() - for k in range(episode_count): - episode=[] - s=self.nn.env(t=t,initial=True) - if self.episode_step==None: - while True: - next_s,r,done,_episode,index=self.env(s,epsilon,t) - self.reward[t]+=r - s=next_s - if type(self.done_pool[t])==np.ndarray: - self._train_(t) - if self.save_episode==True: - try: - if index not in self.finish_list: - episode.append(_episode) - except UnboundLocalError: - pass - if done: - if self.PN==True: - if self.PO==1 or self.PO==3: - self.lock[2].acquire() - else: - self.lock[3].acquire() - else: - self.lock[0].acquire() - self.total_episode+=1 - self.loss_list.append(self.loss[t]) - if self.PN==True: - if self.PO==1 or self.PO==3: - self.lock[2].release() - else: - self.lock[3].release() - else: - self.lock[0].release() - if self.save_episode==True: - episode.append('done') - break - else: - for l in range(self.episode_step): - next_s,r,done,_episode,index=self.env(s,epsilon,t) - self.reward[t]+=r - s=next_s - if type(self.done_pool[t])==np.ndarray: - self._train_(t) - if self.save_episode==True: - try: - if index not in self.finish_list: - episode.append(_episode) - except UnboundLocalError: - pass - if done: - if self.PN==True: - if self.PO==1 or self.PO==3: - self.lock[2].acquire() - else: - self.lock[3].acquire() - else: - self.lock[0].acquire() - self.total_episode+=1 - self.loss_list.append(self.loss[t]) - if self.PN==True: - if self.PO==1 or self.PO==3: - self.lock[2].release() - else: - self.lock[3].release() - else: - self.lock[0].release() - if self.save_episode==True: - episode.append('done') - break - if l==self.episode_step-1: - if self.PN==True: - if self.PO==1 or self.PO==3: - self.lock[2].acquire() - else: - self.lock[3].acquire() - else: - self.lock[0].acquire() - self.total_episode+=1 - self.loss_list.append(self.loss[t]) - if self.PN==True: - if self.PO==1 or self.PO==3: - self.lock[2].release() - else: - self.lock[3].release() - else: - self.lock[0].release() - if self.PN==True: - if self.PO==1 or self.PO==3: - self.lock[2].acquire() - else: - self.lock[3].acquire() - else: - self.lock[0].acquire() - self.reward_list.append(self.reward[t]) - self.reward[t]=0 - if self.save_episode==True: - self.episode_set.append(episode) - if self.max_episode_count!=None and len(self.episode_set)>=self.max_episode_count: - self.save_episode=False - if self.PN==True: - if self.PO==1 or self.PO==3: - self.lock[2].release() - else: - self.lock[3].release() - else: - self.lock[0].release() - try: - if self.nn.row!=None: - pass - except AttributeError: - self.running_flag[t+1]=0 - if self.PO==1 or self.PO==3: - self.lock[2].acquire() - else: - self.lock[3].acquire() - self.process_thread_counter-=1 - if t not in self.finish_list: - self.finish_list[t]=t - if self.PO==1 or self.PO==3: - self.lock[2].release() - else: - self.lock[3].release() - del self.state_pool[t] - del self.action_pool[t] - del self.next_state_pool[t] - del self.reward_pool[t] - del self.done_pool[t] - return + epsilon=self.epsilon[p] # get the epsilon value for the process index from the epsilon array + except Exception: # if any exception occurs + epsilon=None # assign None value to epsilon + if self.PO==1 or self.PO==2: # if the optimization strategy is lock or global lock + lock[1].release() # release the lock for initializing the pools and flags + elif self.PO==3: # if the optimization strategy is no lock + lock[1].release() # release the lock for initializing the pools and flags + for k in range(episode_count): # loop tnrougn episode count + s=self.nn.env(p=p,initial=True) # get the initial state by using the environment method of the neural network model witn the process index and initial flag + if type(self.nn.param[0])!=list: # if the neural network parameter is not a list + s=np.array(s,self.nn.param[0].dtype.name) # convert the state to the same data type as the neural network parameter + else: # if the neural network parameter is a list + s=np.array(s,self.nn.param[0][0].dtype.name) # convert the state to the same data type as the first element of the neural network parameter + if self.episode_step==None: # if episode step value is not given + while True: # loop until episode ends + next_s,r,done,index=self.env(s,epsilon,p,lock,pool_lock) # call the env method witn the state, epsilon value, process index and locks and get the next state, reward, done flag and sampled process index + self.reward[p]+=r # add the reward to the reward array for the process index + s=next_s # assign the next state to the state + if type(self.done_pool[p])==np.ndarray: # if the done flag pool for the process index is a numpy array + self.train_(p,lock,g_lock) # call the train_ method witn the process index and locks to train the neural network model + if done: # if episode ends + if self.PO==1 or self.PO==2: # if the optimization strategy is lock or global lock + lock[1].acquire() # acquire the lock for updating total episode and loss list + self.total_episode.value+=1 # increment total episode by one + self.loss_list.append(self.loss[p]) # append loss array for process index to loss list + if self.PO==1 or self.PO==2: # if optimization strategy is lock or global lock + lock[1].release() # release the lock for updating total episode and loss list + break # break from loop and start next episode + else: # if episode step value is given + for l in range(self.episode_step): # loop through episode step value + next_s,r,done,index=self.env(s,epsilon,p,lock,pool_lock) # call the env method with the state, epsilon value, process index and locks and get the next state, reward, done flag and sampled process index + self.reward[p]+=r # add the reward to the reward array for the process index + s=next_s # assign the next state to the state + if type(self.done_pool[p])==np.ndarray: # if the done flag pool for the process index is a numpy array + self.train_(p,lock,g_lock) # call the train_ method with the process index and locks to train the neural network model + if done: # if episode ends + if self.PO==1 or self.PO==2: # if the optimization strategy is lock or global lock + lock[1].acquire() # acquire the lock for updating total episode and loss list + self.total_episode.value+=1 # increment total episode by one + self.loss_list.append(self.loss[p]) # append loss array for process index to loss list + if self.PO==1 or self.PO==2: # if optimization strategy is lock or global lock + lock[1].release() # release the lock for updating total episode and loss list + break # break from loop and start next episode + if l==self.episode_step-1: # if it is the last step of the episode + if self.PO==1 or self.PO==2: # if the optimization strategy is lock or global lock + lock[1].acquire() # acquire the lock for updating total episode and loss list + self.total_episode.value+=1 # increment total episode by one + self.loss_list.append(self.loss[p]) # append loss array for process index to loss list + if self.PO==1 or self.PO==2: # if optimization strategy is lock or global lock + lock[1].release() # release the lock for updating total episode and loss list + if self.PO==1 or self.PO==2: # if the optimization strategy is lock or global lock + lock[1].acquire() # acquire the lock for updating reward list and reward array + elif len(lock)==3: # if there are three locks + lock[2].acquire() # acquire the third lock for updating reward list and reward array + self.reward_list.append(self.reward[p]) # append the reward array for the process index to the reward list + self.reward[p]=0 # reset the reward array for the process index to zero + if self.PO==1 or self.PO==2: # if the optimization strategy is lock or global lock + lock[1].release() # release the lock for updating reward list and reward array + elif len(lock)==3: # if there are three locks + lock[2].release() # release the third lock for updating reward list and reward array + self.running_flag[p+1]=0 # assign zero value to the running flag list at the position of (process index + 1) to indicate the process is not running + if p not in self.finish_list: # if the process index is not in the finish list + self.finish_list[p]=p # assign the process index to the finish list at the position of process index to indicate the process is finished + if self.PO==1 or self.PO==2: # if the optimization strategy is lock or global lock + lock[1].acquire() # acquire the lock for decrementing process counter + elif self.PO==3: # if the optimization strategy is no lock + lock[1].acquire() # acquire the lock for decrementing process counter + self.process_counter.value-=1 # decrement the process counter by one + if self.PO==1 or self.PO==2: # if the optimization strategy is lock or global lock + lock[1].release() # release the lock for decrementing process counter + elif self.PO==3: # if the optimization strategy is no lock + lock[1].release() # release the lock for decrementing process counter + del self.state_pool[p] # delete the state pool for the process index + del self.action_pool[p] # delete the action pool for the process index + del self.next_state_pool[p] # delete the next state pool for the process index + del self.reward_pool[p] # delete the reward pool for the process index + del self.done_pool[p] # delete the done flag pool for the process index + return # return from the method - def visualize_reward(self): - print() - plt.figure(1) - plt.plot(np.arange(len(self.reward_list)),self.reward_list) - plt.xlabel('episode') - plt.ylabel('reward') - print('reward:{0:.6f}'.format(self.reward_list[-1])) - return + def stop_func(self): # a method to check if the termination condition is met + if self.end(): # call the end method and check if it returns True + self.save(self.total_episode) # call the save method witn total episode to save the neural network model + self.save_flag.value=True # assign True value to save flag to indicate model is saved + self.stop_flag.value=True # assign True value to stop flag to indicate all processes should stop + return True # return True to indicate termination condition is met + return False # return False to indicate termination condition is not met - def visualize_train(self): - print() - plt.figure(1) - plt.plot(np.arange(len(self.loss_list)),self.loss_list) - plt.title('train loss') - plt.xlabel('episode') - plt.ylabel('loss') - print('loss:{0:.6f}'.format(self.loss_list[-1])) - return + def stop_func_(self,lock=None): # a method to check if the stop flag is True and release the lock if needed + if self.stop==True: # if the stop attribute is True + if self.stop_flag.value==True or self.stop_func(): # if the stop flag is True or the stop_func method returns True + if self.PO!=3: # if the optimization strategy is not no lock + lock.release() # release the lock + return True # return True to indicate stop condition is met + return False # return False to indicate stop condition is not met - def visualize_reward_loss(self): - print() - plt.figure(1) - plt.plot(np.arange(len(self.reward_list)),self.reward_list,'r-',label='reward') - plt.plot(np.arange(len(self.loss_list)),self.loss_list,'b-',label='train loss') - plt.xlabel('epoch') - plt.ylabel('reward and loss') - return + def visualize_reward(self): # a method to visualize the reward list as a line plot + print() # print a blank line + plt.figure(1) # create a figure with index 1 + plt.plot(np.arange(len(self.reward_list)),self.reward_list) # plot the reward list as a line with x-axis as the episode index and y-axis as the reward value + plt.xlabel('episode') # set the x-axis label as 'episode' + plt.ylabel('reward') # set the y-axis label as 'reward' + print('reward:{0:.6f}'.format(self.reward_list[-1])) # print the last reward value with six decimal places + return # return from the method + + + def visualize_train(self): # a method to visualize the loss list as a line plot + print() # print a blank line + plt.figure(1) # create a figure with index 1 + plt.plot(np.arange(len(self.loss_list)),self.loss_list) # plot the loss list as a line with x-axis as the episode index and y-axis as the loss value + plt.title('train loss') # set the title of the plot as 'train loss' + plt.xlabel('episode') # set the x-axis label as 'episode' + plt.ylabel('loss') # set the y-axis label as 'loss' + print('loss:{0:.6f}'.format(self.loss_list[-1])) # print the last loss value with six decimal places + return # return from the method + + + def visualize_reward_loss(self): # a method to visualize both the reward list and the loss list as lines on the same plot + print() # print a blank line + plt.figure(1) # create a figure with index 1 + plt.plot(np.arange(len(self.reward_list)),self.reward_list,'r-',label='reward') # plot the reward list as a red line with x-axis as the episode index and y-axis as the reward value and label it as 'reward' + plt.plot(np.arange(len(self.loss_list)),self.loss_list,'b-',label='train loss') # plot the loss list as a blue line witn x-axis as the episode index and y-axis as the loss value and label it as 'train loss' + plt.xlabel('epoch') # set the x-axis label as 'epoch' + plt.ylabel('reward and loss') # set the y-axis label as 'reward and loss' + return # return from the method From eef097d3e5cb9cf0bd61668fa536ddea478b2bd2 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Wed, 28 Jun 2023 20:39:04 +0800 Subject: [PATCH 008/337] Update kernel_reduced.py --- reduced kernel/DL/process/kernel_reduced.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/reduced kernel/DL/process/kernel_reduced.py b/reduced kernel/DL/process/kernel_reduced.py index 5e1e2ead..6561c7f3 100644 --- a/reduced kernel/DL/process/kernel_reduced.py +++ b/reduced kernel/DL/process/kernel_reduced.py @@ -59,10 +59,16 @@ def data(self,train_data=None,train_labels=None,test_data=None,test_labels=None, self.test_flag=True # set the test flag to True self.test_dataset=test_dataset # set the test dataset object self.batch_counter=np.zeros(self.process,dtype=np.int32) # initialize an array to count batches for each process - self.total_loss=np.zeros(self.process,dtype=np.float32) # initialize an array to accumulate loss for each process + if type(self.nn.param[0])!=list: # initialize an array to accumulate loss for each process + self.total_loss=np.zeros(self.process,dtype=self.nn.param[0].dtype.name) + else: + self.total_loss=np.zeros(self.process,dtype=self.nn.param[0][0].dtype.name) try: if self.nn.accuracy!=None: - self.total_acc=np.zeros(self.process,dtype=np.float32) # initialize an array to accumulate accuracy for each process + if type(self.nn.param[0])!=list: # initialize an array to accumulate accuracy for each process + self.total_acc=np.zeros(self.process,dtype=self.nn.param[0].dtype.name) + else: + self.total_acc=np.zeros(self.process,dtype=self.nn.param[0][0].dtype.name) except Exception: pass if self.priority_flag==True: From 88032b2320ba016fcd53d68a4a2449b10139b36b Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Wed, 28 Jun 2023 22:05:44 +0800 Subject: [PATCH 009/337] Delete kernel_reduced.py --- reduced kernel/DL/kernel_reduced.py | 1353 --------------------------- 1 file changed, 1353 deletions(-) delete mode 100644 reduced kernel/DL/kernel_reduced.py diff --git a/reduced kernel/DL/kernel_reduced.py b/reduced kernel/DL/kernel_reduced.py deleted file mode 100644 index 1c06be13..00000000 --- a/reduced kernel/DL/kernel_reduced.py +++ /dev/null @@ -1,1353 +0,0 @@ -from tensorflow import function -import numpy as np -import matplotlib.pyplot as plt -import pickle -import os -import time - - -#You can analyze kernel by example. -''' -example: -import kernel_reduced as k #import kernel -import tensorflow as tf #import platform -import nn as n #import neural network -mnist=tf.keras.datasets.mnist -(x_train,y_train),(x_test,y_test)=mnist.load_data() -x_train,x_test =x_train/255.0,x_test/255.0 -nn=n.nn() #create neural network object -kernel=k.kernel(nn) #start kernel -kernel.platform=tf #use platform -kernel.data(x_train,y_train) #input you data -kernel.train(32,5) #train neural network - #batch size:32 - #epoch:5 - - -multithreading example: -import kernel_reduced as k #import kernel -import tensorflow as tf #import platform -import nn as n #import neural network -import threading -mnist=tf.keras.datasets.mnist -(x_train,y_train),(x_test,y_test)=mnist.load_data() -x_train,x_test =x_train/255.0,x_test/255.0 -nn=n.nn() #create neural network object -kernel=k.kernel(nn) #start kernel -kernel.platform=tf #use platform -kernel.process_thread=7 #thread count,use 7 threads to train -kernel.epoch_=6 #epoch:6 -kernel.PO=2 #use PO2 -kernel.data(x_train,y_train) #input you data -kernel.lock=[threading.Lock(),threading.Lock(),threading.Lock()] -class thread(threading.Thread): - def run(self): - kernel.train(32) #batch size:32 -for _ in range(7): - _thread=thread() - _thread.start() -for _ in range(7): - _thread.join() - - -multiprocessing example: -import kernel_reduced as k #import kernel -import tensorflow as tf #import platform -import nn as n #import neural network -from multiprocessing import Process,Lock -mnist=tf.keras.datasets.mnist -(x_train,y_train),(x_test,y_test)=mnist.load_data() -x_train,x_test =x_train/255.0,x_test/255.0 -nn=n.nn() #create neural network object -kernel=k.kernel(nn) #start kernel -kernel.platform=tf #use platform -kernel.process_thread=7 #thread count,use 7 processes to train -kernel.epoch_=6 #epoch:6 -kernel.PO=2 #use PO2 -kernel.data(x_train,y_train) #input you data -kernel.lock=[Lock(),Lock(),Lock()] -for _ in range(7): - p=Process(target=kernel.train(32)) #batch size:32 - p.start() -for _ in range(7): - p.join() -''' -class kernel: - def __init__(self,nn=None): - self.nn=nn #Neural network object. - self.platform=None #Platform object,kernel use it to distinguish platform you use. - self.PO=None #PO object,three parallel optimization methods correspond to three numbers. - self.lock=None #External incoming lock. - self.process_thread=None #Process thread object,threads or processes count you use. - self.multiprocessing_threading=None - self.process_thread_counter=0 - self.train_ds=None - self.data_segment_flag=False - self.batches=None - self.buffer_size=None - self.epoch_=None #Training epoch count. - self.epoch_counter=0 - self.save_flag=False - self.save_epoch=None - self.batch=None - self.epoch=0 - self.acc_flag='%' #This object be used for acc printing. - self.train_counter=0 - self.opt_counter=None - self.filename='save.dat' #Saving file name. - self.train_loss=None - self.train_acc=None - self.train_loss_list=[] #This object be used for visualization function. - self.train_acc_list=[] #This object be used for visualization function. - self.test_loss=None - self.test_acc=None - self.test_loss_list=[] #This object be used for visualization function. - self.test_acc_list=[] #This object be used for visualization function. - self.test_flag=False #If you have test data,kernel will set it to True. - self.total_epoch=0 - self.time=0 - self.total_time=0 - - - #Turn training data into kernel's instance object. - def data(self,train_data=None,train_labels=None,test_data=None,test_labels=None,train_dataset=None,test_dataset=None): - self.train_data=train_data - self.train_labels=train_labels - self.train_dataset=train_dataset - if self.data_segment_flag==True: - self.train_data,self.train_labels=self.segment_data() - self.test_data=test_data - self.test_labels=test_labels - self.test_dataset=test_dataset - try: #If have test data,kernel set test_flag=True. - if test_data==None: - self.test_flag=False - except ValueError: - self.test_flag=True - if self.train_dataset==None: - if type(self.train_data)==list: - self.shape0=train_data[0].shape[0] - else: - self.shape0=train_data.shape[0] - if self.train_counter==0 and self.process_thread!=None: - if type(self.process_thread)==list: - self.process_thread_num=np.arange(self.process_thread[0]) - self.process_thread_num=list(self.process_thread_num) - self.thread_num=np.arange(self.process_thread[0]*self.process_thread[1]).reshape(self.process_thread[0],self.process_thread[1]) - self.thread_num=list(self.thread_num) - self.thread_num=[list(self.thread_num[i]) for i in range(len(self.thread_num))] - else: - self.process_thread_num=np.arange(self.process_thread) - self.process_thread_num=list(self.process_thread_num) - if self.epoch_!=None: - if type(self.process_thread)==list: - self.batch_counter=np.zeros([self.process_thread[0]*self.process_thread[1]],dtype=np.int32) - self.total_loss=np.zeros([self.process_thread[0]*self.process_thread[1]],dtype=np.float32) - else: - self.batch_counter=np.zeros(self.process_thread,dtype=np.int32) - self.total_loss=np.zeros(self.process_thread,dtype=np.float32) - try: - if self.nn.accuracy!=None: - if type(self.process_thread)==list: - self.total_acc=np.zeros([self.process_thread[0]*self.process_thread[1]],dtype=np.float32) - else: - self.total_acc=np.zeros(self.process_thread,dtype=np.float32) - except AttributeError: - pass - try: - if self.nn.attenuate!=None: #If neural network object defining attenuate function,kernel will initialize opt_counter(optimization counter). - if type(self.process_thread)==list: - self.opt_counter=np.zeros([self.process_thread[0]*self.process_thread[1]],dtype=np.float32) - else: - self.opt_counter=np.zeros(self.process_thread,dtype=np.float32) - except AttributeError: - pass - try: - if type(self.process_thread)==list: - self.nn.bc=np.zeros([self.process_thread[0]*self.process_thread[1]],dtype=np.float32) - else: - self.nn.bc=np.zeros(self.process_thread,dtype=np.float32) - except AttributeError: - pass - return - - - def segment_data(self): - if len(self.train_data)!=self.process_thread: - data=None - labels=None - segments=int((len(self.train_data)-len(self.train_data)%self.process_thread)/self.process_thread) - for i in range(self.process_thread): - index1=i*segments - index2=(i+1)*segments - if i==0: - data=np.expand_dims(self.train_data[index1:index2],axis=0) - labels=np.expand_dims(self.train_labels[index1:index2],axis=0) - else: - data=np.concatenate((data,np.expand_dims(self.train_data[index1:index2],axis=0))) - labels=np.concatenate((labels,np.expand_dims(self.train_labels[index1:index2],axis=0))) - if len(data)%self.process_thread!=0: - segments+=1 - index1=segments*self.process_thread - index2=self.process_thread-(len(self.train_data)-segments*self.process_thread) - data=np.concatenate((data,np.expand_dims(self.train_data[index1:index2],axis=0))) - labels=np.concatenate((labels,np.expand_dims(self.train_labels[index1:index2],axis=0))) - return data,labels - - - #loss_acc function be used for calculating total loss and total acc. - def loss_acc(self,output=None,labels_batch=None,loss=None,test_batch=None,total_loss=None,total_acc=None): - if self.batch!=None: - total_loss+=loss - try: #If neural network object define accuracy function,kernel will use it to calculate accuracy. - if self.nn.accuracy!=None: - batch_acc=self.nn.accuracy(output,labels_batch) - total_acc+=batch_acc - except AttributeError: - pass - return total_loss,total_acc - else: - loss=loss.numpy() - self.train_loss=loss - self.train_loss_list.append(loss) - try: #If neural network object define accuracy function,kernel will use it to calculate accuracy. - if self.nn.accuracy!=None: - acc=self.nn.accuracy(output,self.train_labels) - acc=acc.numpy() - self.train_acc=acc - self.train_acc_list.append(acc) - except AttributeError: - pass - if self.test_flag==True: #If test_flag=True,kernel call test function. - if self.process_thread_t==None: - self.test_loss,self.test_acc=self.test(self.test_data,self.test_labels,test_batch) - else: - self.test_loss,self.test_acc=self.test(test_batch) - self.test_loss_list.append(self.test_loss) - try: - if self.nn.accuracy!=None: - self.test_acc_list.append(self.test_acc) - except AttributeError: - pass - return - - - #data_func functin be used for returning batch data and concatenating data. - def data_func(self,data_batch=None,labels_batch=None,batch=None,index1=None,index2=None,j=None,flag=None): - if flag==None: - if batch!=1: - data_batch=self.train_data[index1:index2] - else: - data_batch=self.train_data[j] - if batch!=1: - labels_batch=self.train_labels[index1:index2] - else: - labels_batch=self.train_labels[j] - else: - try: - data_batch=self.platform.concat([self.train_data[index1:],self.train_data[:index2]],0) - labels_batch=self.platform.concat([self.train_labels[index1:],self.train_labels[:index2]],0) - except: - data_batch=np.concatenate([self.train_data[index1:],self.train_data[:index2]],0) - labels_batch=np.concatenate([self.train_labels[index1:],self.train_labels[:index2]],0) - return data_batch,labels_batch - - - #Optimization subfunction,it be used for opt function,it used optimization function of tensorflow platform. - @function(jit_compile=True) - def tf_opt(self,data,labels): - try: #If neural network object define GradientTape function,kernel will use it or else use other,this design allow you to implement more complicated operations. - if self.nn.GradientTape!=None: - tape,output,loss=self.nn.GradientTape(data,labels) - except AttributeError: - with self.platform.GradientTape(persistent=True) as tape: - try: #If neural network object define one argument value fp(forward propagation) function,kernel will use it or else use other,this design allow you to implement more complicated operations. - output=self.nn.fp(data) - loss=self.nn.loss(output,labels) - except TypeError: - output,loss=self.nn.fp(data,labels) - try: #If neural network object define gradient function,kernel will use it or else use other,this design allow you to implement more complicated operations. - gradient=self.nn.gradient(tape,loss) - except AttributeError: - gradient=tape.gradient(loss,self.nn.param) - try: - self.nn.opt.apply_gradients(zip(gradient,self.nn.param)) - except AttributeError: - self.nn.opt(gradient) - return output,loss - - - #Optimization subfunction,it be used for opt function,it used optimization function of tensorflow platform and parallel optimization. - @function(jit_compile=True) - def tf_opt_t(self,data,labels,t=None,ln=None,u=None): - try: #If neural network object define GradientTape function,kernel will use it or else use other,this design allow you to implement more complicated operations. - if self.nn.GradientTape!=None: - if type(self.process_thread)==list: - tape,output,loss=self.nn.GradientTape(data,labels,u) - else: - tape,output,loss=self.nn.GradientTape(data,labels,t) - except AttributeError: - with self.platform.GradientTape(persistent=True) as tape: - try: - try: #If neural network object define one argument value fp(forward propagation) function,kernel will use it or else use other,this design allow you to implement more complicated operations. - output=self.nn.fp(data) - loss=self.nn.loss(output,labels) - except TypeError: - output,loss=self.nn.fp(data,labels) - except TypeError: - try: - if type(self.process_thread)==list: - output=self.nn.fp(data,u) - else: - output=self.nn.fp(data,t) - loss=self.nn.loss(output,labels) - except TypeError: - if type(self.process_thread)==list: - output,loss=self.nn.fp(data,labels,u) - else: - output,loss=self.nn.fp(data,labels,t) - try: - if self.nn.attenuate!=None: - if type(self.process_thread)==list: - self.opt_counter[u]=0 - else: - self.opt_counter[t]=0 - except AttributeError: - pass - if self.PO==1: - self.lock[0].acquire() - try: #If neural network object define gradient function,kernel will use it or else use other,this design allow you to implement more complicated operations. - gradient=self.nn.gradient(tape,loss) - except AttributeError: - gradient=tape.gradient(loss,self.nn.param) - try: - if self.nn.attenuate!=None: - if type(self.process_thread)==list: - gradient=self.nn.attenuate(gradient,self.opt_counter,u) - else: - gradient=self.nn.attenuate(gradient,self.opt_counter,t) - except AttributeError: - pass - try: - self.nn.opt.apply_gradients(zip(gradient,self.nn.param)) - except AttributeError: - try: - self.nn.opt(gradient) - except TypeError: - self.nn.opt(gradient,t) - try: - if self.nn.attenuate!=None: - self.opt_counter+=1 - except AttributeError: - pass - self.lock[0].release() - elif self.PO==2: - self.lock[0].acquire() - try: #If neural network object define gradient function,kernel will use it or else use other,this design allow you to implement more complicated operations. - gradient=self.nn.gradient(tape,loss) - except AttributeError: - gradient=tape.gradient(loss,self.nn.param) - self.lock[0].release() - self.lock[1].acquire() - try: - if self.nn.attenuate!=None: - if type(self.process_thread)==list: - gradient=self.nn.attenuate(gradient,self.opt_counter,u) - else: - gradient=self.nn.attenuate(gradient,self.opt_counter,t) - except AttributeError: - pass - try: - self.nn.opt.apply_gradients(zip(gradient,self.nn.param)) - except AttributeError: - try: - self.nn.opt(gradient) - except TypeError: - self.nn.opt(gradient,t) - try: - if self.nn.attenuate!=None: - self.opt_counter+=1 - except AttributeError: - pass - self.lock[1].release() - return output,loss - - - #Optimization subfunction,it be used for opt function,it used optimization function of pytorch platform. - def pytorch_opt(self,data,labels): - output=self.nn.fp(data) - loss=self.nn.loss(output,labels) - try: - self.nn.opt.zero_grad() - loss.backward() - self.nn.opt.step() - except: - self.nn.opt(loss) - return output,loss - - - #Main optimization function. - def opt(self,data,labels): - try: - if self.platform.DType!=None: - output,loss=self.tf_opt(data,labels) - except AttributeError: - output,loss=self.pytorch_opt(data,labels) - return output,loss - - - #Main optimization function,it be used for parallel optimization. - def opt_t(self,data,labels,t=None,u=None): - if type(self.process_thread)==list: - output,loss=self.tf_opt_t(data,labels,u=int(u)) - else: - output,loss=self.tf_opt_t(data,labels,int(t)) - return output,loss - - - #Main optimization function,it be used for online training. - def opt_ol(self,data,labels,t,ln=None): - try: - if self.platform.DType!=None: - if self.process_thread!=None: - output,loss=self.tf_opt_t(data,labels,int(t)) - else: - output,loss=self.tf_opt(data,labels) - except AttributeError: - output,loss=self.pytorch_opt(data,labels) - return output,loss - - - #Training subfunction,it be used for train function and no parallel training. - def _train(self,batch=None,_data_batch=None,_labels_batch=None,test_batch=None): - if batch!=None: - total_loss=0 - total_acc=0 - if self.train_dataset!=None: - for data_batch,labels_batch in self.train_dataset: - output,batch_loss=self.opt(data_batch,labels_batch) - total_loss,total_acc=self.loss_acc(output=output,labels_batch=labels_batch,loss=batch_loss,total_loss=total_loss,total_acc=total_acc) - else: - total_loss=0 - total_acc=0 - batches=int((self.shape0-self.shape0%batch)/batch) - for j in range(batches): - index1=j*batch #Use for batch data's index - index2=(j+1)*batch #Use for batch data's index - data_batch,labels_batch=self.data_func(_data_batch,_labels_batch,batch,index1,index2,j) #Return batch data. - output,batch_loss=self.opt(data_batch,labels_batch) - total_loss,total_acc=self.loss_acc(output=output,labels_batch=labels_batch,loss=batch_loss,total_loss=total_loss,total_acc=total_acc) - try: - self.nn.bc=j - except AttributeError: - pass - if self.shape0%batch!=0: - batches+=1 - index1=batches*batch #Use for batch data's index - index2=batch-(self.shape0-batches*batch) #Use for batch data's index - data_batch,labels_batch=self.data_func(_data_batch,_labels_batch,batch,index1,index2,flag=True) #Return batch data after concatenating. - output,batch_loss=self.opt(data_batch,labels_batch) - total_loss,total_acc=self.loss_acc(output=output,labels_batch=labels_batch,loss=batch_loss,total_loss=total_loss,total_acc=total_acc) - try: - self.nn.bc+=1 - except AttributeError: - pass - try: - if self.platform.DType!=None: - loss=total_loss.numpy()/batches - except AttributeError: - loss=total_loss.detach().numpy()/batches - try: - if self.nn.accuracy!=None: - train_acc=total_acc.numpy()/batches - except AttributeError: - pass - self.train_loss=loss - self.train_loss_list.append(loss) - try: - if self.nn.accuracy!=None: - self.train_acc=train_acc - self.train_acc_list.append(train_acc) - except AttributeError: - pass - if self.test_flag==True: #If test_flag=True,kernel call test function. - if self.process_thread_t==None: - self.test_loss,self.test_acc=self.test(self.test_data,self.test_labels,test_batch) - else: - self.test_loss,self.test_acc=self.test(batch=test_batch) - self.test_loss_list.append(self.test_loss) - try: - if self.nn.accuracy!=None: - self.test_acc_list.append(self.test_acc) - except AttributeError: - pass - else: - output,train_loss=self.opt(self.train_data,self.train_labels) - self.loss_acc(output=output,labels_batch=labels_batch,loss=train_loss,test_batch=test_batch,total_loss=total_loss,total_acc=total_acc) - return - - - #Training subfunction,it be used for _train_ function and parallel training. - def train_(self,_data_batch=None,_labels_batch=None,batch=None,batches=None,test_batch=None,index1=None,index2=None,j=None,t=None): - if batch!=None: - if index1==batches*batch: - data_batch,labels_batch=self.data_func(_data_batch,_labels_batch,batch,index1,index2,j,True) #Return batch data after concatenating. - output,batch_loss=self.opt_t(data_batch,labels_batch,t) - try: - self.nn.bc[t]+=1 - except AttributeError: - pass - try: #If neural network object define accuracy function,kernel will use it to calculate accuracy. - if self.nn.accuracy!=None: - batch_acc=self.nn.accuracy(output,labels_batch) - return batch_loss,batch_acc - except AttributeError: - return batch_loss,None - data_batch,labels_batch=self.data_func(_data_batch,_labels_batch,batch,index1,index2,j) #Return batch data. - output,batch_loss=self.opt_t(data_batch,labels_batch,t) - try: - self.nn.bc[t]=j - except AttributeError: - pass - try: #If neural network object define accuracy function,kernel will use it to calculate accuracy. - if self.nn.accuracy!=None: #If neural network object define accuracy function,kernel will use it to calculate accuracy.s - batch_acc=self.nn.accuracy(output,labels_batch) - return batch_loss,batch_acc - except AttributeError: - return batch_loss,None - else: - output,train_loss=self.opt_t(self.train_data,self.train_labels,t) - if self.PO==1 or self.PO==3: - self.lock[1].acquire() - self.total_epoch+=1 - train_loss=train_loss.numpy() - self.train_loss=train_loss - self.train_loss_list.append(train_loss) - try: #If neural network object define accuracy function,kernel will use it to calculate accuracy. - if self.nn.accuracy!=None: - acc=self.nn.accuracy(output,self.train_labels) - acc=acc.numpy() - self.train_acc=acc - self.train_acc_list.append(acc) - except AttributeError: - pass - if self.test_flag==True: #If test_flag=True,kernel call test function. - if self.process_thread_t==None: - self.test_loss,self.test_acc=self.test(self.test_data,self.test_labels,test_batch,t) - else: - self.test_loss,self.test_acc=self.test(batch=test_batch) - self.test_loss_list.append(self.test_loss) - try: - if self.nn.accuracy!=None: - self.test_acc_list.append(self.test_acc) - except AttributeError: - pass - self.lock[1].release() - else: - self.lock[2].acquire() - self.total_epoch+=1 - train_loss=train_loss.numpy() - self.train_loss=train_loss - self.train_loss_list.append(train_loss) - try: #If neural network object define accuracy function,kernel will use it to calculate accuracy. - if self.nn.accuracy!=None: - acc=self.nn.accuracy(output,self.train_labels) - acc=acc.numpy() - self.train_acc=acc - self.train_acc_list.append(acc) - except AttributeError: - pass - if self.test_flag==True: #If test_flag=True,kernel call test function. - if self.process_thread_t==None: - self.test_loss,self.test_acc=self.test(self.test_data,self.test_labels,test_batch,t) - else: - self.test_loss,self.test_acc=self.test(batch=test_batch) - self.test_loss_list.append(self.test_loss) - try: - if self.nn.accuracy!=None: - self.test_acc_list.append(self.test_acc) - except AttributeError: - pass - self.lock[2].release() - return - - - #Training subfunction,it be used for train function and parallel training. - def _train_(self,batch=None,data_batch=None,labels_batch=None,test_batch=None,t=None): - total_loss=0 - total_acc=0 - batches=int((self.shape0-self.shape0%batch)/batch) - if batch!=None: - for j in range(batches): - index1=j*batch #Use for index of batch data - index2=(j+1)*batch #Use for index of batch data - batch_loss,batch_acc=self.train_(data_batch,labels_batch,batch,batches,test_batch,index1,index2,j,t) - try: - if self.nn.accuracy!=None: - total_loss+=batch_loss - total_acc+=batch_acc - except AttributeError: - total_loss+=batch_loss - if self.shape0%batch!=0: - batches+=1 - index1=batches*batch #Use for index of batch data. - index2=batch-(self.shape0-batches*batch) #Use for index of batch data - batch_loss,batch_acc=self.train_(data_batch,labels_batch,batch,batches,test_batch,index1,index2,None,t) - try: - if self.nn.accuracy!=None: - total_loss+=batch_loss - total_acc+=batch_acc - except AttributeError: - total_loss+=batch_loss - loss=total_loss.numpy()/batches - try: #If neural network object define accuracy function,kernel will calculate mean accuracy. - if self.nn.accuracy!=None: - train_acc=total_acc.numpy()/batches - except AttributeError: - pass - if self.PO==1 or self.PO==3: - self.lock[1].acquire() - else: - self.lock[2].acquire() - self.total_epoch+=1 - self.train_loss=loss - self.train_loss_list.append(loss) - try: #If neural network object define accuracy function,kernel append mean accuracy to acc list. - if self.nn.accuracy!=None: - self.train_acc=train_acc - self.train_acc_list.append(train_acc) - except AttributeError: - pass - if self.test_flag==True: #If test_flag=True,kernel call test function. - if self.process_thread_t==None: - self.test_loss,self.test_acc=self.test(self.test_data,self.test_labels,test_batch,t) - else: - self.test_loss,self.test_acc=self.test(batch=test_batch) - self.test_loss_list.append(self.test_loss) - try: - if self.nn.accuracy!=None: - self.test_acc_list.append(self.test_acc) - except AttributeError: - pass - try: - self.nn.ec+=1 - except AttributeError: - pass - if self.PO==1 or self.PO==3: - self.lock[1].release() - else: - self.lock[2].release() - return - else: - batch_loss,batch_acc=self.train_(data_batch,labels_batch,batch,batches,test_batch,index1,index2,j,t) - return - - - #Training subfunction,it be used for train function and parallel training. - def train7(self,train_ds,t,test_batch): - if type(self.process_thread)==list: - if self.PO==1 or self.PO==3: - self.lock[1].acquire() - else: - self.lock[2].acquire() - u=self.thread_num[t].pop(0) - if self.PO==1 or self.PO==3: - self.lock[1].release() - else: - self.lock[2].release() - while True: - for data_batch,labels_batch in train_ds: - if type(self.process_thread)==list: - output,batch_loss=self.opt_t(data_batch,labels_batch,u=u) - else: - output,batch_loss=self.opt_t(data_batch,labels_batch,t) - try: - if type(self.process_thread)==list: - self.nn.bc[u]+=1 - else: - self.nn.bc[t]+=1 - except AttributeError: - pass - try: - if self.nn.accuracy!=None: - batch_acc=self.nn.accuracy(output,labels_batch) - except AttributeError: - pass - try: - if self.nn.accuracy!=None: - if type(self.process_thread)==list: - self.total_loss[u]+=batch_loss - self.total_acc[u]+=batch_acc - else: - self.total_loss[t]+=batch_loss - self.total_acc[t]+=batch_acc - except AttributeError: - if type(self.process_thread)==list: - self.total_loss[u]+=batch_loss - else: - self.total_loss[t]+=batch_loss - if type(self.process_thread)==list: - self.batch_counter[u]+=1 - else: - self.batch_counter[t]+=1 - if self.PO==1 or self.PO==3: - self.lock[1].acquire() - else: - self.lock[2].acquire() - batches=np.sum(self.batch_counter) - if batches>=self.batches: - self.batch_counter=self.batch_counter*0 - loss=np.sum(self.total_loss)/batches - try: - if self.nn.accuracy!=None: - train_acc=np.sum(self.total_acc)/batches - except AttributeError: - pass - self.total_epoch+=1 - self.train_loss=loss - self.train_loss_list.append(loss) - try: - if self.nn.accuracy!=None: - self.train_acc=train_acc - self.train_acc_list.append(train_acc) - except AttributeError: - pass - if self.test_flag==True: - if self.process_thread_t==None: - self.test_loss,self.test_acc=self.test(self.test_data,self.test_labels,test_batch,t) - else: - self.test_loss,self.test_acc=self.test(batch=test_batch) - self.test_loss_list.append(self.test_loss) - try: - if self.nn.accuracy!=None: - self.test_acc_list.append(self.test_acc) - except AttributeError: - pass - self.epoch_counter+=1 - try: - if type(self.process_thread)==list: - self.nn.bc[u]=0 - else: - self.nn.bc[t]=0 - except AttributeError: - pass - try: - self.nn.ec+=1 - except AttributeError: - pass - self.total_loss=self.total_loss*0 - try: - if self.nn.accuracy!=None: - self.total_acc=self.total_acc*0 - except AttributeError: - pass - if self.PO==1 or self.PO==3: - self.lock[1].release() - else: - self.lock[2].release() - if self.epoch_counter==self.epoch_: - return - - - #Main training function. - def train(self,batch=None,epoch=None,test_batch=None,save=None,one=True,p=None,s=None): - if self.process_thread!=None: - if self.PO==1 or self.PO==3: - self.lock[1].acquire() - else: - self.lock[2].acquire() - t=self.process_thread_num.pop(0) - self.process_thread_counter+=1 - if self.epoch_!=None: - if self.train_dataset==None: - if t==0: - if self.batches==None: - self.batches=int((self.shape0-self.shape0%batch)/batch) - if self.shape0%batch!=0: - self.batches+=1 - if self.PO==1 or self.PO==3: - self.lock[1].release() - else: - self.lock[2].release() - self.batch=batch - self.epoch=0 - self.train_counter+=1 - if self.epoch_!=None: - if self.train_dataset!=None: - train_ds=self.train_dataset - else: - if self.data_segment_flag==True: - train_ds=self.platform.data.Dataset.from_tensor_slices((self.train_data[t],self.train_labels[t])).batch(batch) - elif self.buffer_size!=None: - train_ds=self.platform.data.Dataset.from_tensor_slices((self.train_data,self.train_labels)).shuffle(self.buffer_size).batch(batch) - else: - train_ds=self.platform.data.Dataset.from_tensor_slices((self.train_data,self.train_labels)).batch(batch) - if p==None: #p and self.p variable be used for controlling printing count. - self.p=9 - else: - self.p=p-1 - if s==None: #s and self.s variable be used for controlling saving files. - self.s=1 - self.file_list=None - else: - self.s=s-1 - self.file_list=[] - data_batch=None - labels_batch=None - if self.process_thread!=None and self.epoch_!=None: - if self.multiprocessing_threading==None: - self.train7(train_ds,t,test_batch) - else: - train7=self.train7 - class thread(self.multiprocessing_threading.Thread): - def run(self): - train7(train_ds,t,test_batch) - for _ in range(self.process_thread[1]): - _thread=thread() - _thread.start() - for _ in range(self.process_thread[1]): - _thread.join() - elif epoch!=None: - for i in range(epoch): - t1=time.time() - if self.process_thread==None: - self._train(batch,data_batch,labels_batch,test_batch) - else: - self._train_(batch,data_batch,labels_batch,test_batch,t) - if self.process_thread==None: - try: - self.nn.ec+=1 - except AttributeError: - pass - if self.process_thread==None: - self.total_epoch+=1 - if self.process_thread==None: - if epoch%10!=0: - p=epoch-epoch%self.p - p=int(p/self.p) - s=epoch-epoch%self.s - s=int(s/self.s) - else: - p=epoch/(self.p+1) - p=int(p) - s=epoch/(self.s+1) - s=int(s) - if p==0: - p=1 - if s==0: - s=1 - if i%p==0: - if self.test_flag==False: - try: - if self.nn.accuracy!=None: - print('epoch:{0} loss:{1:.6f}'.format(i+1,self.train_loss)) - if self.acc_flag=='%': - print('epoch:{0} accuracy:{1:.1f}'.format(i+1,self.train_acc*100)) - else: - print('epoch:{0} accuracy:{1:.6f}'.format(i+1,self.train_acc)) - print() - except AttributeError: - print('epoch:{0} loss:{1:.6f}'.format(i+1,self.train_loss)) - print() - else: - try: - if self.nn.accuracy!=None: - print('epoch:{0} loss:{1:.6f},test loss:{2:.6f}'.format(i+1,self.train_loss,self.test_loss)) - if self.acc_flag=='%': - print('epoch:{0} accuracy:{1:.1f},test accuracy:{2:.1f}'.format(i+1,self.train_acc*100,self.test_acc*100)) - else: - print('epoch:{0} accuracy:{1:.1f},test accuracy:{2:.1f}'.format(i+1,self.train_acc,self.test_acc)) - print() - except AttributeError: - print('epoch:{0} loss:{1:.6f},test loss:{2:.6f}'.format(i+1,self.train_loss,self.test_loss)) - print() - if save!=None and i%s==0: - self.save(self.total_epoch,one) - t2=time.time() - if self.process_thread==None: - self.time+=(t2-t1) - else: - i=0 - while True: - t1=time.time() - if self.process_thread==None: - self._train(test_batch=test_batch) - else: - self._train_(test_batch=test_batch,t=t) - i+=1 - if self.process_thread==None: - try: - self.nn.ec+=1 - except AttributeError: - pass - if self.process_thread==None: - self.total_epoch+=1 - if self.process_thread==None: - if epoch%10!=0: - p=epoch-epoch%self.p - p=int(p/self.p) - s=epoch-epoch%self.s - s=int(s/self.s) - else: - p=epoch/(self.p+1) - p=int(p) - s=epoch/(self.s+1) - s=int(s) - if p==0: - p=1 - if s==0: - s=1 - if i%p==0: - if self.test_flag==False: - try: - if self.nn.accuracy!=None: - print('epoch:{0} loss:{1:.6f}'.format(i+1,self.train_loss)) - if self.acc_flag=='%': - print('epoch:{0} accuracy:{1:.1f}'.format(i+1,self.train_acc*100)) - else: - print('epoch:{0} accuracy:{1:.6f}'.format(i+1,self.train_acc)) - print() - except AttributeError: - print('epoch:{0} loss:{1:.6f}'.format(i+1,self.train_loss)) - print() - else: - try: - if self.nn.accuracy!=None: - print('epoch:{0} loss:{1:.6f},test loss:{2:.6f}'.format(i+1,self.train_loss,self.test_loss)) - if self.acc_flag=='%': - print('epoch:{0} accuracy:{1:.1f},test accuracy:{2:.1f}'.format(i+1,self.train_acc*100,self.test_acc*100)) - else: - print('epoch:{0} accuracy:{1:.1f},test accuracy:{2:.1f}'.format(i+1,self.train_acc,self.test_acc)) - print() - except AttributeError: - print('epoch:{0} loss:{1:.6f},test loss:{2:.6f}'.format(i+1,self.train_loss,self.test_loss)) - print() - if save!=None and i%s==0: - self.save(self.total_epoch,one) - t2=time.time() - if self.process_thread==None: - self.time+=(t2-t1) - if save!=None: - self.save() - if self.process_thread==None: - self._time=self.time-int(self.time) - if self._time<0.5: - self.time=int(self.time) - else: - self.time=int(self.time)+1 - self.total_time+=self.time - if self.process_thread==None: - if self.test_flag==False: - print('last loss:{0:.6f}'.format(self.train_loss)) - else: - print('last loss:{0:.6f},last test loss:{1:.6f}'.format(self.train_loss,self.test_loss)) - try: - if self.nn.accuracy!=None: - if self.acc_flag=='%': - if self.test_flag==False: - print('last accuracy:{0:.1f}'.format(self.train_acc*100)) - else: - print('last accuracy:{0:.1f},last test accuracy:{1:.1f}'.format(self.train_acc*100,self.test_acc*100)) - else: - if self.test_flag==False: - print('last accuracy:{0:.6f}'.format(self.train_acc)) - else: - print('last accuracy:{0:.6f},last test accuracy:{1:.6f}'.format(self.train_acc,self.test_acc)) - except AttributeError: - pass - print() - print('time:{0}s'.format(self.time)) - if self.process_thread==None: - self.training_flag=False - if self.process_thread!=None: - if self.PO==1 or self.PO==3: - self.lock[1].acquire() - else: - self.lock[2].acquire() - self.process_thread_counter-=1 - if self.PO==1 or self.PO==3: - self.lock[1].release() - else: - self.lock[2].release() - return - - - def train_ol(self): - if self.process_thread!=None: - if self.PO==1 or self.PO==3: - self.lock[1].acquire() - else: - self.lock[2].acquire() - t=self.process_thread_num.pop(0) - self.process_thread_counter+=1 - if self.PO==1 or self.PO==3: - self.lock[1].release() - else: - self.lock[2].release() - while True: - if self.process_thread!=None: - if self.save_flag==True: - if self.PO==1 or self.PO==3: - self.lock[1].acquire() - else: - self.lock[2].acquire() - self.save() - if self.PO==1 or self.PO==3: - self.lock[1].release() - else: - self.lock[2].release() - try: - data=self.nn.ol(t) - except: - self.exception_list[t]=True - continue - try: - output,loss=self.opt_ol(data[0],data[1],t) - except: - continue - loss=loss.numpy() - if self.PO==1 or self.PO==3: - self.lock[1].acquire() - else: - self.lock[2].acquire() - if len(self.nn.train_loss_list)==self.nn.max_length: - del self.nn.train_loss_list[0] - self.nn.train_loss_list.append(loss) - try: - if self.nn.accuracy!=None: - try: - train_acc=self.nn.accuracy(output,data[1]) - except: - continue - if len(self.nn.train_acc_list)==self.nn.max_length: - del self.nn.train_acc_list[0] - self.train_acc_list.append(train_acc) - except AttributeError: - pass - try: - self.nn.c+=1 - except AttributeError: - pass - if self.PO==1 or self.PO==3: - self.lock[1].release() - else: - self.lock[2].release() - else: - if self.save_flag==True: - self.save() - data=self.nn.ol() - if data=='stop': - return - output,loss=self.opt(data[0],data[1]) - loss=loss.numpy() - if len(self.nn.train_loss_list)==self.nn.max_length: - del self.nn.train_loss_list[0] - self.nn.train_loss_list.append(loss) - try: - if self.nn.accuracy!=None: - train_acc=self.nn.accuracy(output,data[1]) - if len(self.nn.train_acc_list)==self.nn.max_length: - del self.nn.train_acc_list[0] - self.train_acc_list.append(train_acc) - except AttributeError: - pass - try: - self.nn.c+=1 - except AttributeError: - pass - if self.process_thread!=None: - if self.PO==1 or self.PO==3: - self.lock[1].acquire() - else: - self.lock[2].acquire() - self.process_thread_counter-=1 - if self.PO==1 or self.PO==3: - self.lock[1].release() - else: - self.lock[2].release() - return - - - def test(self,test_data=None,test_labels=None,batch=None,t=None): - if type(test_data)==list: - data_batch=[x for x in range(len(test_data))] - if type(test_labels)==list: - labels_batch=[x for x in range(len(test_labels))] - if batch!=None: - total_loss=0 - total_acc=0 - if self.test_dataset!=None: - for data_batch,labels_batch in self.test_dataset: - if self.process_thread==None or t==None: - output=self.nn.fp(data_batch) - else: - output=self.nn.fp(data_batch,t) - batch_loss=self.nn.loss(output,labels_batch) - total_loss+=batch_loss - try: - if self.nn.accuracy!=None: - batch_acc=self.nn.accuracy(output,labels_batch) - total_acc+=batch_acc - except AttributeError: - pass - else: - total_loss=0 - total_acc=0 - if type(test_data)==list: - batches=int((test_data[0].shape[0]-test_data[0].shape[0]%batch)/batch) - shape0=test_data[0].shape[0] - else: - batches=int((test_data.shape[0]-test_data.shape[0]%batch)/batch) - shape0=test_data.shape[0] - for j in range(batches): - index1=j*batch - index2=(j+1)*batch - if type(test_data)==list: - for i in range(len(test_data)): - data_batch[i]=test_data[i][index1:index2] - else: - data_batch=test_data[index1:index2] - if type(test_labels)==list: - for i in range(len(test_labels)): - labels_batch[i]=test_labels[i][index1:index2] - else: - labels_batch=test_labels[index1:index2] - if self.process_thread==None or t==None: - output=self.nn.fp(data_batch) - else: - output=self.nn.fp(data_batch,t) - batch_loss=self.nn.loss(output,labels_batch) - total_loss+=batch_loss - try: - if self.nn.accuracy!=None: - batch_acc=self.nn.accuracy(output,labels_batch) - total_acc+=batch_acc - except AttributeError: - pass - if shape0%batch!=0: - batches+=1 - index1=batches*batch - index2=batch-(shape0-batches*batch) - try: - if type(test_data)==list: - for i in range(len(test_data)): - data_batch[i]=self.platform.concat([test_data[i][index1:],test_data[i][:index2]],0) - else: - data_batch=self.platform.concat([test_data[index1:],test_data[:index2]],0) - if type(self.test_labels)==list: - for i in range(len(test_labels)): - labels_batch[i]=self.platform.concat([test_labels[i][index1:],test_labels[i][:index2]],0) - else: - labels_batch=self.platform.concat([test_labels[index1:],test_labels[:index2]],0) - except: - if type(test_data)==list: - for i in range(len(test_data)): - data_batch[i]=self.platform.concat([test_data[i][index1:],test_data[i][:index2]],0) - else: - data_batch=self.platform.concat([test_data[index1:],test_data[:index2]],0) - if type(self.test_labels)==list: - for i in range(len(test_labels)): - labels_batch[i]=self.platform.concat([test_labels[i][index1:],test_labels[i][:index2]],0) - else: - labels_batch=self.platform.concat([test_labels[index1:],test_labels[:index2]],0) - if self.process_thread==None or t==None: - output=self.nn.fp(data_batch) - else: - output=self.nn.fp(data_batch,t) - batch_loss=self.nn.loss(output,labels_batch) - total_loss+=batch_loss - try: - if self.nn.accuracy!=None: - batch_acc=self.nn.accuracy(output,labels_batch) - total_acc+=batch_acc - except AttributeError: - pass - test_loss=total_loss.numpy()/batches - try: - if self.nn.accuracy!=None: - test_acc=total_acc.numpy()/batches - except AttributeError: - pass - else: - if self.process_thread==None or t==None: - output=self.nn.fp(test_data) - else: - output=self.nn.fp(test_data,t) - test_loss=self.nn.loss(output,test_labels) - test_loss=test_loss.numpy() - try: - if self.nn.accuracy!=None: - test_acc=self.nn.accuracy(output,test_labels) - test_acc=test_acc.numpy() - except AttributeError: - pass - try: - if self.nn.accuracy!=None: - return test_loss,test_acc - except AttributeError: - return test_loss,None - - - def visualize_train(self): - print() - plt.figure(1) - plt.plot(np.arange(self.total_epoch),self.train_loss_list) - plt.title('train loss') - plt.xlabel('epoch') - plt.ylabel('loss') - print('train loss:{0:.6f}'.format(self.train_loss)) - try: - if self.nn.accuracy!=None: - plt.figure(2) - plt.plot(np.arange(self.total_epoch),self.train_acc_list) - plt.title('train acc') - plt.xlabel('epoch') - plt.ylabel('acc') - if self.acc_flag=='%': - print('train acc:{0:.1f}'.format(self.train_acc*100)) - else: - print('train acc:{0:.6f}'.format(self.train_acc)) - except AttributeError: - pass - return - - - def visualize_test(self): - print() - plt.figure(1) - plt.plot(np.arange(self.total_epoch),self.test_loss_list) - plt.title('test loss') - plt.xlabel('epoch') - plt.ylabel('loss') - print('test loss:{0:.6f}'.format(self.test_loss)) - try: - if self.nn.accuracy!=None: - plt.figure(2) - plt.plot(np.arange(self.total_epoch),self.test_acc_list) - plt.title('test acc') - plt.xlabel('epoch') - plt.ylabel('acc') - if self.acc_flag=='%': - print('test acc:{0:.1f}'.format(self.test_acc*100)) - else: - print('test acc:{0:.6f}'.format(self.test_acc)) - except AttributeError: - pass - return - - - def comparison(self): - print() - plt.figure(1) - plt.plot(np.arange(self.total_epoch),self.train_loss_list,'b-',label='train loss') - if self.test_flag==True: - plt.plot(np.arange(self.total_epoch),self.test_loss_list,'r-',label='test loss') - plt.xlabel('epoch') - plt.ylabel('loss') - print('train loss:{0}'.format(self.train_loss)) - plt.legend() - try: - if self.nn.accuracy!=None: - plt.figure(2) - plt.plot(np.arange(self.total_epoch),self.train_acc_list,'b-',label='train acc') - if self.test_flag==True: - plt.plot(np.arange(self.total_epoch),self.test_acc_list,'r-',label='test acc') - plt.xlabel('epoch') - plt.ylabel('acc') - plt.legend() - if self.acc_flag=='%': - print('train acc:{0:.1f}'.format(self.train_acc*100)) - else: - print('train acc:{0:.6f}'.format(self.train_acc)) - except AttributeError: - pass - if self.test_flag==True: - print() - print('-------------------------------------') - print() - print('test loss:{0:.6f}'.format(self.test_loss)) - if self.acc_flag=='%': - print('test acc:{0:.1f}'.format(self.test_acc*100)) - else: - print('test acc:{0:.6f}'.format(self.test_acc)) - return - - - def save_p(self): - parameter_file=open('param.dat','wb') - pickle.dump(self.nn.param,parameter_file) - parameter_file.close() - return - - - def save(self,i=None,one=True): - if one==True: - output_file=open(self.filename,'wb') - else: - filename=self.filename.replace(self.filename[self.filename.find('.'):],'-{0}.dat'.format(i)) - output_file=open(filename,'wb') - self.file_list.append([filename]) - if len(self.file_list)>self.s+1: - os.remove(self.file_list[0][0]) - del self.file_list[0] - try: - pickle.dump(self.nn,output_file) - except: - opt=self.nn.opt - self.nn.opt=None - pickle.dump(self.nn,output_file) - self.nn.opt=opt - try: - pickle.dump(self.platform.keras.optimizers.serialize(opt),output_file) - except: - pickle.dump(self.nn.serialize(),output_file) - else: - pickle.dump(None,output_file) - pickle.dump(self.batch,output_file) - pickle.dump(self.acc_flag,output_file) - pickle.dump(self.file_list,output_file) - pickle.dump(self.train_counter,output_file) - pickle.dump(self.train_loss,output_file) - pickle.dump(self.train_acc,output_file) - pickle.dump(self.train_loss_list,output_file) - pickle.dump(self.train_acc_list,output_file) - pickle.dump(self.test_flag,output_file) - if self.test_flag==True: - pickle.dump(self.test_loss,output_file) - pickle.dump(self.test_acc,output_file) - pickle.dump(self.test_loss_list,output_file) - pickle.dump(self.test_acc_list,output_file) - pickle.dump(self.total_epoch,output_file) - pickle.dump(self.total_time,output_file) - output_file.close() - return - - - def restore(self,s_path): - input_file=open(s_path,'rb') - self.nn=pickle.load(input_file) - opt_serialized=pickle.load(input_file) - try: - self.nn.opt=self.platform.keras.optimizers.deserialize(opt_serialized) - except: - self.nn.deserialize(opt_serialized) - else: - pass - self.batch=pickle.load(input_file) - self.acc_flag=pickle.load(input_file) - self.file_list=pickle.load(input_file) - self.train_counter=pickle.load(input_file) - self.train_loss=pickle.load(input_file) - self.train_acc=pickle.load(input_file) - self.train_loss_list=pickle.load(input_file) - self.train_acc_list=pickle.load(input_file) - self.test_flag=pickle.load(input_file) - if self.test_flag==True: - self.test_loss=pickle.load(input_file) - self.test_acc=pickle.load(input_file) - self.test_loss_list=pickle.load(input_file) - self.test_acc_list=pickle.load(input_file) - self.total_epoch=pickle.load(input_file) - self.total_time=pickle.load(input_file) - input_file.close() - return From 5cdf0f24cd0b5f089f8126e5db9255a0599da24e Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Wed, 28 Jun 2023 22:05:51 +0800 Subject: [PATCH 010/337] Delete kernel_reduced_no_parallel.py --- .../DL/kernel_reduced_no_parallel.py | 647 ------------------ 1 file changed, 647 deletions(-) delete mode 100644 reduced kernel/DL/kernel_reduced_no_parallel.py diff --git a/reduced kernel/DL/kernel_reduced_no_parallel.py b/reduced kernel/DL/kernel_reduced_no_parallel.py deleted file mode 100644 index fc21c5d9..00000000 --- a/reduced kernel/DL/kernel_reduced_no_parallel.py +++ /dev/null @@ -1,647 +0,0 @@ -from tensorflow import function -import numpy as np -import matplotlib.pyplot as plt -import pickle -import os -import time - - -#Doesn't support parallelism -#You can analyze kernel by example. -''' -example: -import kernel_reduced_no_parallel as k #import kernel -import tensorflow as tf #import platform -import nn as n #import neural network -mnist=tf.keras.datasets.mnist -(x_train,y_train),(x_test,y_test)=mnist.load_data() -x_train,x_test =x_train/255.0,x_test/255.0 -nn=n.nn() #create neural network object -kernel=k.kernel(nn) #start kernel -kernel.platform=tf #use platform -kernel.data(x_train,y_train) #input you data -kernel.train(32,5) #train neural network - #batch size:32 - #epoch:5 -''' -class kernel: - def __init__(self,nn=None): - self.nn=nn #Neural network object. - self.platform=None #Platform object,kernel use it to distinguish platform you use. - self.batch=None - self.epoch=0 - self.acc_flag='%' #This object be used for acc printing. - self.train_counter=0 - self.filename='save.dat' #Saving file name. - self.train_loss=None - self.train_acc=None - self.train_loss_list=[] #This object be used for visualization function. - self.train_acc_list=[] #This object be used for visualization function. - self.test_loss=None - self.test_acc=None - self.test_loss_list=[] #This object be used for visualization function. - self.test_acc_list=[] #This object be used for visualization function. - self.test_flag=False #If you have test data,kernel will set it to True. - self.total_epoch=0 - self.time=0 - self.total_time=0 - - - #Turn training data into kernel's instance object. - def data(self,train_data=None,train_labels=None,test_data=None,test_labels=None,train_dataset=None,test_dataset=None): - self.train_data=train_data - self.train_labels=train_labels - self.train_dataset=train_dataset - self.test_data=test_data - self.test_labels=test_labels - self.test_dataset=test_dataset - try: #If have test data,kernel set test_flag=True. - if test_data==None: - self.test_flag=False - except ValueError: - self.test_flag=True - if self.train_dataset==None: - self.shape0=train_data.shape[0] - return - - - #loss_acc function be used for calculating total loss and total acc. - def loss_acc(self,output=None,labels_batch=None,loss=None,test_batch=None,total_loss=None,total_acc=None): - if self.batch!=None: - total_loss+=loss - try: #If neural network object define accuracy function,kernel will use it to calculate accuracy. - if self.nn.accuracy!=None: - batch_acc=self.nn.accuracy(output,labels_batch) - total_acc+=batch_acc - except AttributeError: - pass - return total_loss,total_acc - else: - loss=loss.numpy() - self.train_loss=loss - self.train_loss_list.append(loss) - try: #If neural network object define accuracy function,kernel will use it to calculate accuracy. - if self.nn.accuracy!=None: - acc=self.nn.accuracy(output,self.train_labels) - acc=acc.numpy() - self.train_acc=acc - self.train_acc_list.append(acc) - except AttributeError: - pass - if self.test_flag==True: #If test_flag=True,kernel call test function. - self.test_loss,self.test_acc=self.test(self.test_data,self.test_labels,test_batch) - self.test_loss_list.append(self.test_loss) - try: - if self.nn.accuracy!=None: - self.test_acc_list.append(self.test_acc) - except AttributeError: - pass - return - - - #data_func functin be used for returning batch data and concatenating data. - def data_func(self,data_batch=None,labels_batch=None,batch=None,index1=None,index2=None,j=None,flag=None): - if flag==None: - if batch!=1: - data_batch=self.train_data[index1:index2] - else: - data_batch=self.train_data[j] - if batch!=1: - labels_batch=self.train_labels[index1:index2] - else: - labels_batch=self.train_labels[j] - else: - try: - data_batch=self.platform.concat([self.train_data[index1:],self.train_data[:index2]],0) - labels_batch=self.platform.concat([self.train_labels[index1:],self.train_labels[:index2]],0) - except: - data_batch=np.concatenate([self.train_data[index1:],self.train_data[:index2]],0) - labels_batch=np.concatenate([self.train_labels[index1:],self.train_labels[:index2]],0) - return data_batch,labels_batch - - - #Optimization subfunction,it be used for opt function,it used optimization function of tensorflow platform. - @function(jit_compile=True) - def tf_opt(self,data,labels): - try: #If neural network object define GradientTape function,kernel will use it or else use other,this design allow you to implement more complicated operations. - if self.nn.GradientTape!=None: - tape,output,loss=self.nn.GradientTape(data,labels) - except AttributeError: - with self.platform.GradientTape(persistent=True) as tape: - try: #If neural network object define one argument value fp(forward propagation) function,kernel will use it or else use other,this design allow you to implement more complicated operations. - output=self.nn.fp(data) - loss=self.nn.loss(output,labels) - except TypeError: - output,loss=self.nn.fp(data,labels) - try: #If neural network object define gradient function,kernel will use it or else use other,this design allow you to implement more complicated operations. - gradient=self.nn.gradient(tape,loss) - except AttributeError: - gradient=tape.gradient(loss,self.nn.param) - try: - self.nn.opt.apply_gradients(zip(gradient,self.nn.param)) - except AttributeError: - self.nn.opt(gradient) - return output,loss - - - #Optimization subfunction,it be used for opt function,it used optimization function of pytorch platform. - def pytorch_opt(self,data,labels): - output=self.nn.fp(data) - loss=self.nn.loss(output,labels) - try: - self.nn.opt.zero_grad() - loss.backward() - self.nn.opt.step() - except: - self.nn.opt(loss) - return output,loss - - - #Main optimization function. - def opt(self,data,labels): - try: - if self.platform.DType!=None: - output,loss=self.tf_opt(data,labels) - except AttributeError: - output,loss=self.pytorch_opt(data,labels) - return output,loss - - - #Training subfunction,it be used for train function and no parallel training. - def _train(self,batch=None,_data_batch=None,_labels_batch=None,test_batch=None): - if batch!=None: - total_loss=0 - total_acc=0 - if self.train_dataset!=None: - for data_batch,labels_batch in self.train_dataset: - output,batch_loss=self.opt(data_batch,labels_batch) - total_loss,total_acc=self.loss_acc(output=output,labels_batch=labels_batch,loss=batch_loss,total_loss=total_loss,total_acc=total_acc) - else: - total_loss=0 - total_acc=0 - batches=int((self.shape0-self.shape0%batch)/batch) - for j in range(batches): - index1=j*batch #Use for index of batch data - index2=(j+1)*batch #Use for index of batch data - data_batch,labels_batch=self.data_func(_data_batch,_labels_batch,batch,index1,index2,j) #Return batch data. - output,batch_loss=self.opt(data_batch,labels_batch) - total_loss,total_acc=self.loss_acc(output=output,labels_batch=labels_batch,loss=batch_loss,total_loss=total_loss,total_acc=total_acc) - try: - self.nn.bc=j - except AttributeError: - pass - if self.shape0%batch!=0: - batches+=1 - index1=batches*batch #Use for index of batch data - index2=batch-(self.shape0-batches*batch) #Use for index of batch data - data_batch,labels_batch=self.data_func(_data_batch,_labels_batch,batch,index1,index2,flag=True) #Return batch data after concatenating. - output,batch_loss=self.opt(data_batch,labels_batch) - total_loss,total_acc=self.loss_acc(output=output,labels_batch=labels_batch,loss=batch_loss,total_loss=total_loss,total_acc=total_acc) - try: - self.nn.bc+=1 - except AttributeError: - pass - try: - if self.platform.DType!=None: - loss=total_loss.numpy()/batches - except AttributeError: - loss=total_loss.detach().numpy()/batches - try: #If neural network object define accuracy function,kernel will calculate mean accuracy. - if self.nn.accuracy!=None: - train_acc=total_acc.numpy()/batches - except AttributeError: - pass - self.train_loss=loss - self.train_loss_list.append(loss) - try: #If neural network object define accuracy function,kernel append mean accuracy to acc list. - if self.nn.accuracy!=None: - self.train_acc=train_acc - self.train_acc_list.append(train_acc) - except AttributeError: - pass - if self.test_flag==True: #If test_flag=True,kernel call test function. - self.test_loss,self.test_acc=self.test(self.test_data,self.test_labels,test_batch) - self.test_loss_list.append(self.test_loss) - try: - if self.nn.accuracy!=None: - self.test_acc_list.append(self.test_acc) - except AttributeError: - pass - else: - output,train_loss=self.opt(self.train_data,self.train_labels) - self.loss_acc(output=output,labels_batch=labels_batch,loss=train_loss,test_batch=test_batch,total_loss=total_loss,total_acc=total_acc) - return - - - #Main training function. - def train(self,batch=None,epoch=None,test_batch=None,save=None,one=True,p=None,s=None): - self.batch=batch - self.epoch=0 - self.train_counter+=1 - if p==None: #p and self.p variable be used for controlling printing count. - self.p=9 - else: - self.p=p-1 - if s==None: #s and self.s variable be used for controlling saving files. - self.s=1 - self.file_list=None - else: - self.s=s-1 - self.file_list=[] - data_batch=None - labels_batch=None - if epoch!=None: - for i in range(epoch): - t1=time.time() - self._train(batch,data_batch,labels_batch,test_batch) - try: - self.nn.ec+=1 - except AttributeError: - pass - self.total_epoch+=1 - if epoch%10!=0: - p=epoch-epoch%self.p - p=int(p/self.p) - s=epoch-epoch%self.s - s=int(s/self.s) - else: - p=epoch/(self.p+1) - p=int(p) - s=epoch/(self.s+1) - s=int(s) - if p==0: - p=1 - if s==0: - s=1 - if i%p==0: - if self.test_flag==False: - try: - if self.nn.accuracy!=None: - print('epoch:{0} loss:{1:.6f}'.format(i+1,self.train_loss)) - if self.acc_flag=='%': - print('epoch:{0} accuracy:{1:.1f}'.format(i+1,self.train_acc*100)) - else: - print('epoch:{0} accuracy:{1:.6f}'.format(i+1,self.train_acc)) - print() - except AttributeError: - print('epoch:{0} loss:{1:.6f}'.format(i+1,self.train_loss)) - print() - else: - try: - if self.nn.accuracy!=None: - print('epoch:{0} loss:{1:.6f},test loss:{2:.6f}'.format(i+1,self.train_loss,self.test_loss)) - if self.acc_flag=='%': - print('epoch:{0} accuracy:{1:.1f},test accuracy:{2:.1f}'.format(i+1,self.train_acc*100,self.test_acc*100)) - else: - print('epoch:{0} accuracy:{1:.1f},test accuracy:{2:.1f}'.format(i+1,self.train_acc,self.test_acc)) - print() - except AttributeError: - print('epoch:{0} loss:{1:.6f},test loss:{2:.6f}'.format(i+1,self.train_loss,self.test_loss)) - print() - if save!=None and i%s==0: - self.save(self.total_epoch,one) - t2=time.time() - self.time+=(t2-t1) - else: - i=0 - while True: - t1=time.time() - self._train(test_batch=test_batch) - i+=1 - try: - self.nn.ec+=1 - except AttributeError: - pass - self.total_epoch+=1 - if epoch%10!=0: - p=epoch-epoch%self.p - p=int(p/self.p) - s=epoch-epoch%self.s - s=int(s/self.s) - else: - p=epoch/(self.p+1) - p=int(p) - s=epoch/(self.s+1) - s=int(s) - if p==0: - p=1 - if s==0: - s=1 - if i%p==0: - if self.test_flag==False: - try: - if self.nn.accuracy!=None: - print('epoch:{0} loss:{1:.6f}'.format(i+1,self.train_loss)) - if self.acc_flag=='%': - print('epoch:{0} accuracy:{1:.1f}'.format(i+1,self.train_acc*100)) - else: - print('epoch:{0} accuracy:{1:.6f}'.format(i+1,self.train_acc)) - print() - except AttributeError: - print('epoch:{0} loss:{1:.6f}'.format(i+1,self.train_loss)) - print() - else: - try: - if self.nn.accuracy!=None: - print('epoch:{0} loss:{1:.6f},test loss:{2:.6f}'.format(i+1,self.train_loss,self.test_loss)) - if self.acc_flag=='%': - print('epoch:{0} accuracy:{1:.1f},test accuracy:{2:.1f}'.format(i+1,self.train_acc*100,self.test_acc*100)) - else: - print('epoch:{0} accuracy:{1:.1f},test accuracy:{2:.1f}'.format(i+1,self.train_acc,self.test_acc)) - print() - except AttributeError: - print('epoch:{0} loss:{1:.6f},test loss:{2:.6f}'.format(i+1,self.train_loss,self.test_loss)) - print() - if save!=None and i%s==0: - self.save(self.total_epoch,one) - t2=time.time() - self.time+=(t2-t1) - if save!=None: - self.save() - self._time=self.time-int(self.time) - if self._time<0.5: - self.time=int(self.time) - else: - self.time=int(self.time)+1 - self.total_time+=self.time - if self.test_flag==False: - print('last loss:{0:.6f}'.format(self.train_loss)) - else: - print('last loss:{0:.6f},last test loss:{1:.6f}'.format(self.train_loss,self.test_loss)) - try: - if self.nn.accuracy!=None: - if self.acc_flag=='%': - if self.test_flag==False: - print('last accuracy:{0:.1f}'.format(self.train_acc*100)) - else: - print('last accuracy:{0:.1f},last test accuracy:{1:.1f}'.format(self.train_acc*100,self.test_acc*100)) - else: - if self.test_flag==False: - print('last accuracy:{0:.6f}'.format(self.train_acc)) - else: - print('last accuracy:{0:.6f},last test accuracy:{1:.6f}'.format(self.train_acc,self.test_acc)) - except AttributeError: - pass - print() - print('time:{0}s'.format(self.time)) - self.training_flag=False - return - - - def test(self,test_data=None,test_labels=None,batch=None): - if type(test_data)==list: - data_batch=[x for x in range(len(test_data))] - if type(test_labels)==list: - labels_batch=[x for x in range(len(test_labels))] - if batch!=None: - total_loss=0 - total_acc=0 - if type(test_data)==list: - batches=int((test_data[0].shape[0]-test_data[0].shape[0]%batch)/batch) - shape0=test_data[0].shape[0] - else: - batches=int((test_data.shape[0]-test_data.shape[0]%batch)/batch) - shape0=test_data.shape[0] - for j in range(batches): - index1=j*batch - index2=(j+1)*batch - if type(test_data)==list: - for i in range(len(test_data)): - data_batch[i]=test_data[i][index1:index2] - else: - data_batch=test_data[index1:index2] - if type(test_labels)==list: - for i in range(len(test_labels)): - labels_batch[i]=test_labels[i][index1:index2] - else: - labels_batch=test_labels[index1:index2] - output=self.nn.fp(data_batch) - batch_loss=self.nn.loss(output,labels_batch) - total_loss+=batch_loss - try: - if self.nn.accuracy!=None: - batch_acc=self.nn.accuracy(output,labels_batch) - total_acc+=batch_acc - except AttributeError: - pass - if shape0%batch!=0: - batches+=1 - index1=batches*batch - index2=batch-(shape0-batches*batch) - try: - if type(test_data)==list: - for i in range(len(test_data)): - data_batch[i]=self.platform.concat([test_data[i][index1:],test_data[i][:index2]],0) - else: - data_batch=self.platform.concat([test_data[index1:],test_data[:index2]],0) - if type(self.test_labels)==list: - for i in range(len(test_labels)): - labels_batch[i]=self.platform.concat([test_labels[i][index1:],test_labels[i][:index2]],0) - else: - labels_batch=self.platform.concat([test_labels[index1:],test_labels[:index2]],0) - except: - if type(test_data)==list: - for i in range(len(test_data)): - data_batch[i]=self.platform.concat([test_data[i][index1:],test_data[i][:index2]],0) - else: - data_batch=self.platform.concat([test_data[index1:],test_data[:index2]],0) - if type(self.test_labels)==list: - for i in range(len(test_labels)): - labels_batch[i]=self.platform.concat([test_labels[i][index1:],test_labels[i][:index2]],0) - else: - labels_batch=self.platform.concat([test_labels[index1:],test_labels[:index2]],0) - output=self.nn.fp(data_batch) - batch_loss=self.nn.loss(output,labels_batch) - total_loss+=batch_loss - try: - if self.nn.accuracy!=None: - batch_acc=self.nn.accuracy(output,labels_batch) - total_acc+=batch_acc - except AttributeError: - pass - test_loss=total_loss.numpy()/batches - try: - if self.nn.accuracy!=None: - test_acc=total_acc.numpy()/batches - except AttributeError: - pass - else: - output=self.nn.fp(test_data) - test_loss=self.nn.loss(output,test_labels) - test_loss=test_loss.numpy() - try: - if self.nn.accuracy!=None: - test_acc=self.nn.accuracy(output,test_labels) - test_acc=test_acc.numpy() - except AttributeError: - pass - try: - if self.nn.accuracy!=None: - return test_loss,test_acc - except AttributeError: - return test_loss,None - - - def visualize_train(self): - print() - plt.figure(1) - plt.plot(np.arange(self.total_epoch),self.train_loss_list) - plt.title('train loss') - plt.xlabel('epoch') - plt.ylabel('loss') - print('train loss:{0:.6f}'.format(self.train_loss)) - try: - if self.nn.accuracy!=None: - plt.figure(2) - plt.plot(np.arange(self.total_epoch),self.train_acc_list) - plt.title('train acc') - plt.xlabel('epoch') - plt.ylabel('acc') - if self.acc_flag=='%': - print('train acc:{0:.1f}'.format(self.train_acc*100)) - else: - print('train acc:{0:.6f}'.format(self.train_acc)) - except AttributeError: - pass - return - - - def visualize_test(self): - print() - plt.figure(1) - plt.plot(np.arange(self.total_epoch),self.test_loss_list) - plt.title('test loss') - plt.xlabel('epoch') - plt.ylabel('loss') - print('test loss:{0:.6f}'.format(self.test_loss)) - try: - if self.nn.accuracy!=None: - plt.figure(2) - plt.plot(np.arange(self.total_epoch),self.test_acc_list) - plt.title('test acc') - plt.xlabel('epoch') - plt.ylabel('acc') - if self.acc_flag=='%': - print('test acc:{0:.1f}'.format(self.test_acc*100)) - else: - print('test acc:{0:.6f}'.format(self.test_acc)) - except AttributeError: - pass - return - - - def comparison(self): - print() - plt.figure(1) - plt.plot(np.arange(self.total_epoch),self.train_loss_list,'b-',label='train loss') - if self.test_flag==True: - plt.plot(np.arange(self.total_epoch),self.test_loss_list,'r-',label='test loss') - plt.xlabel('epoch') - plt.ylabel('loss') - print('train loss:{0}'.format(self.train_loss)) - plt.legend() - try: - if self.nn.accuracy!=None: - plt.figure(2) - plt.plot(np.arange(self.total_epoch),self.train_acc_list,'b-',label='train acc') - if self.test_flag==True: - plt.plot(np.arange(self.total_epoch),self.test_acc_list,'r-',label='test acc') - plt.xlabel('epoch') - plt.ylabel('acc') - plt.legend() - if self.acc_flag=='%': - print('train acc:{0:.1f}'.format(self.train_acc*100)) - else: - print('train acc:{0:.6f}'.format(self.train_acc)) - except AttributeError: - pass - if self.test_flag==True: - print() - print('-------------------------------------') - print() - print('test loss:{0:.6f}'.format(self.test_loss)) - if self.acc_flag=='%': - print('test acc:{0:.1f}'.format(self.test_acc*100)) - else: - print('test acc:{0:.6f}'.format(self.test_acc)) - return - - - def save_p(self): - parameter_file=open('param.dat','wb') - pickle.dump(self.nn.param,parameter_file) - parameter_file.close() - return - - - def save(self,i=None,one=True): - if one==True: - output_file=open(self.filename,'wb') - else: - filename=self.filename.replace(self.filename[self.filename.find('.'):],'-{0}.dat'.format(i)) - output_file=open(filename,'wb') - self.file_list.append([filename]) - if len(self.file_list)>self.s+1: - os.remove(self.file_list[0][0]) - del self.file_list[0] - try: - pickle.dump(self.nn,output_file) - except: - opt=self.nn.opt - self.nn.opt=None - pickle.dump(self.nn,output_file) - self.nn.opt=opt - try: - pickle.dump(self.platform.keras.optimizers.serialize(opt),output_file) - except: - pickle.dump(self.nn.serialize(),output_file) - else: - pickle.dump(None,output_file) - pickle.dump(self.batch,output_file) - pickle.dump(self.acc_flag,output_file) - pickle.dump(self.file_list,output_file) - pickle.dump(self.train_counter,output_file) - pickle.dump(self.train_loss,output_file) - pickle.dump(self.train_acc,output_file) - pickle.dump(self.train_loss_list,output_file) - pickle.dump(self.train_acc_list,output_file) - pickle.dump(self.test_flag,output_file) - if self.test_flag==True: - pickle.dump(self.test_loss,output_file) - pickle.dump(self.test_acc,output_file) - pickle.dump(self.test_loss_list,output_file) - pickle.dump(self.test_acc_list,output_file) - pickle.dump(self.total_epoch,output_file) - pickle.dump(self.total_time,output_file) - output_file.close() - return - - - def restore(self,s_path): - input_file=open(s_path,'rb') - self.nn=pickle.load(input_file) - opt_serialized=pickle.load(input_file) - try: - self.nn.opt=self.platform.keras.optimizers.deserialize(opt_serialized) - except: - self.nn.deserialize(opt_serialized) - else: - pass - self.batch=pickle.load(input_file) - self.acc_flag=pickle.load(input_file) - self.file_list=pickle.load(input_file) - self.train_counter=pickle.load(input_file) - self.train_loss=pickle.load(input_file) - self.train_acc=pickle.load(input_file) - self.train_loss_list=pickle.load(input_file) - self.train_acc_list=pickle.load(input_file) - self.test_flag=pickle.load(input_file) - if self.test_flag==True: - self.test_loss=pickle.load(input_file) - self.test_acc=pickle.load(input_file) - self.test_loss_list=pickle.load(input_file) - self.test_acc_list=pickle.load(input_file) - self.total_epoch=pickle.load(input_file) - self.total_time=pickle.load(input_file) - input_file.close() - return From 0cd8a71812e6ac7e45a2b27fb3e306ae5db383b9 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Wed, 28 Jun 2023 22:05:58 +0800 Subject: [PATCH 011/337] Delete kernel_reduced_parallel.py --- reduced kernel/DL/kernel_reduced_parallel.py | 832 ------------------- 1 file changed, 832 deletions(-) delete mode 100644 reduced kernel/DL/kernel_reduced_parallel.py diff --git a/reduced kernel/DL/kernel_reduced_parallel.py b/reduced kernel/DL/kernel_reduced_parallel.py deleted file mode 100644 index 88d5f86d..00000000 --- a/reduced kernel/DL/kernel_reduced_parallel.py +++ /dev/null @@ -1,832 +0,0 @@ -from tensorflow import function -import numpy as np -import matplotlib.pyplot as plt - - -#Keep only parallel part. -#You can analyze kernel by example. -''' -multithreading example: -import kernel_reduced_parallel as k #import kernel -import tensorflow as tf #import platform -import nn as n #import neural network -import threading -mnist=tf.keras.datasets.mnist -(x_train,y_train),(x_test,y_test)=mnist.load_data() -x_train,x_test =x_train/255.0,x_test/255.0 -nn=n.nn() #create neural network object -kernel=k.kernel(nn) #start kernel -kernel.platform=tf #use platform -kernel.process_thread=7 #thread count,use 7 threads to train -kernel.epoch_=6 #epoch:6 -kernel.PO=2 #use PO2 -kernel.data(x_train,y_train) #input you data -kernel.lock=[threading.Lock(),threading.Lock(),threading.Lock()] -class thread(threading.Thread): - def run(self): - kernel.train(32) #batch size:32 -for _ in range(7): - _thread=thread() - _thread.start() -for _ in range(7): - _thread.join() - - -multiprocessing example: -import kernel_reduced_parallel as k #import kernel -import tensorflow as tf #import platform -import nn as n #import neural network -from multiprocessing import Process,Lock -mnist=tf.keras.datasets.mnist -(x_train,y_train),(x_test,y_test)=mnist.load_data() -x_train,x_test =x_train/255.0,x_test/255.0 -nn=n.nn() #create neural network object -kernel=k.kernel(nn) #start kernel -kernel.platform=tf #use platform -kernel.process_thread=7 #thread count,use 7 processes to train -kernel.epoch_=6 #epoch:6 -kernel.PO=2 #use PO2 -kernel.data(x_train,y_train) #input you data -kernel.lock=[Lock(),Lock(),Lock()] -for _ in range(7): - p=Process(target=kernel.train(32)) #batch size:32 - p.start() -for _ in range(7): - p.join() -''' -class kernel: - def __init__(self,nn=None): - self.nn=nn #Neural network object. - self.platform=None #Platform object,kernel use it to distinguish platform you use. - self.PO=None #PO object,three parallel optimization methods correspond to three numbers. - self.lock=None #External incoming lock. - self.process_thread=None #Process thread object,threads or processes count you use. - self.multiprocessing_threading=None - self.process_thread_counter=0 - self.train_ds=None - self.data_segment_flag=False - self.batches=None - self.buffer_size=None - self.epoch_=None #Training epoch count. - self.epoch_counter=0 - self.batch=None - self.epoch=0 - self.acc_flag='%' #This object be used for acc printing. - self.train_counter=0 - self.opt_counter=None - self.train_loss=None - self.train_acc=None - self.train_loss_list=[] #This object be used for visualization function. - self.train_acc_list=[] #This object be used for visualization function. - self.test_loss=None - self.test_acc=None - self.test_loss_list=[] #This object be used for visualization function. - self.test_acc_list=[] #This object be used for visualization function. - self.test_flag=False #If you have test data,kernel will set it to True. - self.total_epoch=0 - - - #Turn training data into kernel's instance object. - def data(self,train_data=None,train_labels=None,test_data=None,test_labels=None,train_dataset=None,test_dataset=None): - self.train_data=train_data - self.train_labels=train_labels - self.train_dataset=train_dataset - if self.data_segment_flag==True: - self.train_data,self.train_labels=self.segment_data() - self.test_data=test_data - self.test_labels=test_labels - self.test_dataset=test_dataset - try: #If have test data,kernel set test_flag=True. - if test_data==None: - self.test_flag=False - except ValueError: - self.test_flag=True - if self.train_dataset==None: - if type(self.train_data)==list: - self.shape0=train_data[0].shape[0] - else: - self.shape0=train_data.shape[0] - if self.train_counter==0 and self.process_thread!=None: - if type(self.process_thread)==list: - self.process_thread_num=np.arange(self.process_thread[0]) - self.process_thread_num=list(self.process_thread_num) - self.thread_num=np.arange(self.process_thread[0]*self.process_thread[1]).reshape(self.process_thread[0],self.process_thread[1]) - self.thread_num=list(self.thread_num) - self.thread_num=[list(self.thread_num[i]) for i in range(len(self.thread_num))] - else: - self.process_thread_num=np.arange(self.process_thread) - self.process_thread_num=list(self.process_thread_num) - if self.epoch_!=None: - if type(self.process_thread)==list: - self.batch_counter=np.zeros([self.process_thread[0]*self.process_thread[1]],dtype=np.int32) - self.total_loss=np.zeros([self.process_thread[0]*self.process_thread[1]],dtype=np.float32) - else: - self.batch_counter=np.zeros(self.process_thread,dtype=np.int32) - self.total_loss=np.zeros(self.process_thread,dtype=np.float32) - try: - if self.nn.accuracy!=None: - if type(self.process_thread)==list: - self.total_acc=np.zeros([self.process_thread[0]*self.process_thread[1]],dtype=np.float32) - else: - self.total_acc=np.zeros(self.process_thread,dtype=np.float32) - except AttributeError: - pass - try: #If neural network object defining attenuate function,kernel will initialize opt_counter(optimization counter). - if self.nn.attenuate!=None: - if type(self.process_thread)==list: - self.opt_counter=np.zeros([self.process_thread[0]*self.process_thread[1]],dtype=np.float32) - else: - self.opt_counter=np.zeros(self.process_thread,dtype=np.float32) - except AttributeError: - pass - try: - if type(self.process_thread)==list: - self.nn.bc=np.zeros([self.process_thread[0]*self.process_thread[1]],dtype=np.float32) - else: - self.nn.bc=np.zeros(self.process_thread,dtype=np.float32) - except AttributeError: - pass - return - - - def segment_data(self): - if len(self.train_data)!=self.process_thread: - data=None - labels=None - segments=int((len(self.train_data)-len(self.train_data)%self.process_thread)/self.process_thread) - for i in range(self.process_thread): - index1=i*segments - index2=(i+1)*segments - if i==0: - data=np.expand_dims(self.train_data[index1:index2],axis=0) - labels=np.expand_dims(self.train_labels[index1:index2],axis=0) - else: - data=np.concatenate((data,np.expand_dims(self.train_data[index1:index2],axis=0))) - labels=np.concatenate((labels,np.expand_dims(self.train_labels[index1:index2],axis=0))) - if len(data)%self.process_thread!=0: - segments+=1 - index1=segments*self.process_thread - index2=self.process_thread-(len(self.train_data)-segments*self.process_thread) - data=np.concatenate((data,np.expand_dims(self.train_data[index1:index2],axis=0))) - labels=np.concatenate((labels,np.expand_dims(self.train_labels[index1:index2],axis=0))) - return data,labels - - - #data_func functin be used for returning batch data and concatenating data. - def data_func(self,data_batch=None,labels_batch=None,batch=None,index1=None,index2=None,j=None,flag=None): - if flag==None: - if batch!=1: - data_batch=self.train_data[index1:index2] - else: - data_batch=self.train_data[j] - if batch!=1: - labels_batch=self.train_labels[index1:index2] - else: - labels_batch=self.train_labels[j] - else: - try: - data_batch=self.platform.concat([self.train_data[index1:],self.train_data[:index2]],0) - labels_batch=self.platform.concat([self.train_labels[index1:],self.train_labels[:index2]],0) - except: - data_batch=np.concatenate([self.train_data[index1:],self.train_data[:index2]],0) - labels_batch=np.concatenate([self.train_labels[index1:],self.train_labels[:index2]],0) - return data_batch,labels_batch - - - #Optimization subfunction,it be used for opt function,it used optimization function of tensorflow platform and parallel optimization. - @function(jit_compile=True) - def tf_opt_t(self,data,labels,t=None,ln=None,u=None): - try: #If neural network object define GradientTape function,kernel will use it or else use other,this design allow you to implement more complicated operations. - if self.nn.GradientTape!=None: - if type(self.process_thread)==list: - tape,output,loss=self.nn.GradientTape(data,labels,u) - else: - tape,output,loss=self.nn.GradientTape(data,labels,t) - except AttributeError: - with self.platform.GradientTape(persistent=True) as tape: - try: - try: #If neural network object define one argument value fp(forward propagation) function,kernel will use it or else use other,this design allow you to implement more complicated operations. - output=self.nn.fp(data) - loss=self.nn.loss(output,labels) - except TypeError: - output,loss=self.nn.fp(data,labels) - except TypeError: - try: - if type(self.process_thread)==list: - output=self.nn.fp(data,u) - else: - output=self.nn.fp(data,t) - loss=self.nn.loss(output,labels) - except TypeError: - if type(self.process_thread)==list: - output,loss=self.nn.fp(data,labels,u) - else: - output,loss=self.nn.fp(data,labels,t) - try: - if self.nn.attenuate!=None: - if type(self.process_thread)==list: - self.opt_counter[u]=0 - else: - self.opt_counter[t]=0 - except AttributeError: - pass - if self.PO==1: - self.lock[0].acquire() - try: #If neural network object define gradient function,kernel will use it or else use other,this design allow you to implement more complicated operations. - gradient=self.nn.gradient(tape,loss) - except AttributeError: - gradient=tape.gradient(loss,self.nn.param) - try: - if self.nn.attenuate!=None: - if type(self.process_thread)==list: - gradient=self.nn.attenuate(gradient,self.opt_counter,u) - else: - gradient=self.nn.attenuate(gradient,self.opt_counter,t) - except AttributeError: - pass - try: - self.nn.opt.apply_gradients(zip(gradient,self.nn.param)) - except AttributeError: - try: - self.nn.opt(gradient) - except TypeError: - self.nn.opt(gradient,t) - try: - if self.nn.attenuate!=None: - self.opt_counter+=1 - except AttributeError: - pass - self.lock[0].release() - elif self.PO==2: - self.lock[0].acquire() - try: #If neural network object define gradient function,kernel will use it or else use other,this design allow you to implement more complicated operations. - gradient=self.nn.gradient(tape,loss) - except AttributeError: - gradient=tape.gradient(loss,self.nn.param) - self.lock[0].release() - self.lock[1].acquire() - try: - if self.nn.attenuate!=None: - if type(self.process_thread)==list: - gradient=self.nn.attenuate(gradient,self.opt_counter,u) - else: - gradient=self.nn.attenuate(gradient,self.opt_counter,t) - except AttributeError: - pass - try: - self.nn.opt.apply_gradients(zip(gradient,self.nn.param)) - except AttributeError: - try: - self.nn.opt(gradient) - except TypeError: - self.nn.opt(gradient,t) - try: - if self.nn.attenuate!=None: - self.opt_counter+=1 - except AttributeError: - pass - self.lock[1].release() - return output,loss - - - #Main optimization function,it be used for parallel optimization. - def opt_t(self,data,labels,t=None,u=None): - if type(self.process_thread)==list: - output,loss=self.tf_opt_t(data,labels,u=int(u)) - else: - output,loss=self.tf_opt_t(data,labels,int(t)) - return output,loss - - - #Training subfunction,it be used for _train_ function and parallel training. - def train_(self,_data_batch=None,_labels_batch=None,batch=None,batches=None,test_batch=None,index1=None,index2=None,j=None,t=None): - if batch!=None: - if index1==batches*batch: - data_batch,labels_batch=self.data_func(_data_batch,_labels_batch,batch,index1,index2,j,True) #Return batch data after concatenating. - output,batch_loss=self.opt_t(data_batch,labels_batch,t) - try: - self.nn.bc[t]+=1 - except AttributeError: - pass - try: #If neural network object define accuracy function,kernel will use it to calculate accuracy. - if self.nn.accuracy!=None: - batch_acc=self.nn.accuracy(output,labels_batch) - return batch_loss,batch_acc - except AttributeError: - return batch_loss,None - data_batch,labels_batch=self.data_func(_data_batch,_labels_batch,batch,index1,index2,j) #Return batch data. - output,batch_loss=self.opt_t(data_batch,labels_batch,t) - try: - self.nn.bc[t]=j - except AttributeError: - pass - try: #If neural network object define accuracy function,kernel will use it to calculate accuracy. - if self.nn.accuracy!=None: - batch_acc=self.nn.accuracy(output,labels_batch) - return batch_loss,batch_acc - except AttributeError: - return batch_loss,None - else: - output,train_loss=self.opt_t(self.train_data,self.train_labels,t) - if self.PO==1 or self.PO==3: - self.lock[1].acquire() - self.total_epoch+=1 - train_loss=train_loss.numpy() - self.train_loss=train_loss - self.train_loss_list.append(train_loss) - try: #If neural network object define accuracy function,kernel will use it to calculate accuracy. - if self.nn.accuracy!=None: - acc=self.nn.accuracy(output,self.train_labels) - acc=acc.numpy() - self.train_acc=acc - self.train_acc_list.append(acc) - except AttributeError: - pass - if self.test_flag==True: - if self.process_thread_t==None: - self.test_loss,self.test_acc=self.test(self.test_data,self.test_labels,test_batch,t) - else: - self.test_loss,self.test_acc=self.test(batch=test_batch) - self.test_loss_list.append(self.test_loss) - try: - if self.nn.accuracy!=None: - self.test_acc_list.append(self.test_acc) - except AttributeError: - pass - self.lock[1].release() - else: - self.lock[2].acquire() - self.total_epoch+=1 - train_loss=train_loss.numpy() - self.train_loss=train_loss - self.train_loss_list.append(train_loss) - try: #If neural network object define accuracy function,kernel will use it to calculate accuracy. - if self.nn.accuracy!=None: - acc=self.nn.accuracy(output,self.train_labels) - acc=acc.numpy() - self.train_acc=acc - self.train_acc_list.append(acc) - except AttributeError: - pass - if self.test_flag==True: #If test_flag=True,kernel call test function. - if self.process_thread_t==None: - self.test_loss,self.test_acc=self.test(self.test_data,self.test_labels,test_batch,t) - else: - self.test_loss,self.test_acc=self.test(batch=test_batch) - self.test_loss_list.append(self.test_loss) - try: - if self.nn.accuracy!=None: - self.test_acc_list.append(self.test_acc) - except AttributeError: - pass - self.lock[2].release() - return - - - #Training subfunction,it be used for train function and parallel training. - def _train_(self,batch=None,data_batch=None,labels_batch=None,test_batch=None,t=None): - total_loss=0 - total_acc=0 - batches=int((self.shape0-self.shape0%batch)/batch) - if batch!=None: - for j in range(batches): - index1=j*batch #Use for index of batch data - index2=(j+1)*batch #Use for index of batch data - batch_loss,batch_acc=self.train_(data_batch,labels_batch,batch,batches,test_batch,index1,index2,j,t) - try: - if self.nn.accuracy!=None: - total_loss+=batch_loss - total_acc+=batch_acc - except AttributeError: - total_loss+=batch_loss - if self.shape0%batch!=0: - batches+=1 - index1=batches*batch #Use for index of batch data - index2=batch-(self.shape0-batches*batch) #Use for index of batch data - batch_loss,batch_acc=self.train_(data_batch,labels_batch,batch,batches,test_batch,index1,index2,None,t) - try: - if self.nn.accuracy!=None: - total_loss+=batch_loss - total_acc+=batch_acc - except AttributeError: - total_loss+=batch_loss - loss=total_loss.numpy()/batches - try: #If neural network object define accuracy function,kernel will calculate mean accuracy. - if self.nn.accuracy!=None: - train_acc=total_acc.numpy()/batches - except AttributeError: - pass - if self.PO==1 or self.PO==3: - self.lock[1].acquire() - else: - self.lock[2].acquire() - self.total_epoch+=1 - self.train_loss=loss - self.train_loss_list.append(loss) - try: #If neural network object define accuracy function,kernel append mean accuracy to acc list. - if self.nn.accuracy!=None: - self.train_acc=train_acc - self.train_acc_list.append(train_acc) - except AttributeError: - pass - if self.test_flag==True: #If test_flag=True,kernel call test function. - if self.process_thread_t==None: - self.test_loss,self.test_acc=self.test(self.test_data,self.test_labels,test_batch,t) - else: - self.test_loss,self.test_acc=self.test(batch=test_batch) - self.test_loss_list.append(self.test_loss) - try: - if self.nn.accuracy!=None: - self.test_acc_list.append(self.test_acc) - except AttributeError: - pass - try: - self.nn.ec+=1 - except AttributeError: - pass - if self.PO==1 or self.PO==3: - self.lock[1].release() - else: - self.lock[2].release() - return - else: - batch_loss,batch_acc=self.train_(data_batch,labels_batch,batch,batches,test_batch,index1,index2,j,t) - return - - - #Training subfunction,it be used for train function and parallel training. - def train7(self,train_ds,t,test_batch): - if type(self.process_thread)==list: - if self.PO==1 or self.PO==3: - self.lock[1].acquire() - else: - self.lock[2].acquire() - u=self.thread_num[t].pop(0) - if self.PO==1 or self.PO==3: - self.lock[1].release() - else: - self.lock[2].release() - while True: - for data_batch,labels_batch in train_ds: - if type(self.process_thread)==list: - output,batch_loss=self.opt_t(data_batch,labels_batch,u=u) - else: - output,batch_loss=self.opt_t(data_batch,labels_batch,t) - try: - if type(self.process_thread)==list: - self.nn.bc[u]+=1 - else: - self.nn.bc[t]+=1 - except AttributeError: - pass - try: - if self.nn.accuracy!=None: - batch_acc=self.nn.accuracy(output,labels_batch) - except AttributeError: - pass - try: - if self.nn.accuracy!=None: - if type(self.process_thread)==list: - self.total_loss[u]+=batch_loss - self.total_acc[u]+=batch_acc - else: - self.total_loss[t]+=batch_loss - self.total_acc[t]+=batch_acc - except AttributeError: - if type(self.process_thread)==list: - self.total_loss[u]+=batch_loss - else: - self.total_loss[t]+=batch_loss - if type(self.process_thread)==list: - self.batch_counter[u]+=1 - else: - self.batch_counter[t]+=1 - if self.PO==1 or self.PO==3: - self.lock[1].acquire() - else: - self.lock[2].acquire() - batches=np.sum(self.batch_counter) - if batches>=self.batches: - self.batch_counter=self.batch_counter*0 - loss=np.sum(self.total_loss)/batches - try: - if self.nn.accuracy!=None: - train_acc=np.sum(self.total_acc)/batches - except AttributeError: - pass - self.total_epoch+=1 - self.train_loss=loss - self.train_loss_list.append(loss) - try: - if self.nn.accuracy!=None: - self.train_acc=train_acc - self.train_acc_list.append(train_acc) - except AttributeError: - pass - if self.test_flag==True: - if self.process_thread_t==None: - self.test_loss,self.test_acc=self.test(self.test_data,self.test_labels,test_batch,t) - else: - self.test_loss,self.test_acc=self.test(batch=test_batch) - self.test_loss_list.append(self.test_loss) - try: - if self.nn.accuracy!=None: - self.test_acc_list.append(self.test_acc) - except AttributeError: - pass - self.epoch_counter+=1 - try: - if type(self.process_thread)==list: - self.nn.bc[u]=0 - else: - self.nn.bc[t]=0 - except AttributeError: - pass - try: - self.nn.ec+=1 - except AttributeError: - pass - self.total_loss=self.total_loss*0 - try: - if self.nn.accuracy!=None: - self.total_acc=self.total_acc*0 - except AttributeError: - pass - if self.PO==1 or self.PO==3: - self.lock[1].release() - else: - self.lock[2].release() - if self.epoch_counter==self.epoch_: - return - - - #Main training function. - def train(self,batch=None,epoch=None,test_batch=None): - if self.process_thread!=None: - if self.PO==1 or self.PO==3: - self.lock[1].acquire() - else: - self.lock[2].acquire() - t=self.process_thread_num.pop(0) - self.process_thread_counter+=1 - if self.epoch_!=None: - if self.train_dataset==None: - if t==0: - if self.batches==None: - self.batches=int((self.shape0-self.shape0%batch)/batch) - if self.shape0%batch!=0: - self.batches+=1 - if self.PO==1 or self.PO==3: - self.lock[1].release() - else: - self.lock[2].release() - self.batch=batch - self.epoch=0 - self.train_counter+=1 - if self.epoch_!=None: - if self.train_dataset!=None: - train_ds=self.train_dataset - else: - if self.data_segment_flag==True: - train_ds=self.platform.data.Dataset.from_tensor_slices((self.train_data[t],self.train_labels[t])).batch(batch) - elif self.buffer_size!=None: - train_ds=self.platform.data.Dataset.from_tensor_slices((self.train_data,self.train_labels)).shuffle(self.buffer_size).batch(batch) - else: - train_ds=self.platform.data.Dataset.from_tensor_slices((self.train_data,self.train_labels)).batch(batch) - data_batch=None - labels_batch=None - if self.process_thread!=None and self.epoch_!=None: - if self.multiprocessing_threading==None: - self.train7(train_ds,t,test_batch) - else: - train7=self.train7 - class thread(self.multiprocessing_threading.Thread): - def run(self): - train7(train_ds,t,test_batch) - for _ in range(self.process_thread[1]): - _thread=thread() - _thread.start() - for _ in range(self.process_thread[1]): - _thread.join() - elif epoch!=None: - for i in range(epoch): - self._train_(batch,data_batch,labels_batch,test_batch,t) - else: - while True: - self._train_(test_batch=test_batch,t=t) - if self.process_thread!=None: - if self.PO==1 or self.PO==3: - self.lock[1].acquire() - else: - self.lock[2].acquire() - self.process_thread_counter-=1 - if self.PO==1 or self.PO==3: - self.lock[1].release() - else: - self.lock[2].release() - return - - - def test(self,test_data=None,test_labels=None,batch=None,t=None): - if type(test_data)==list: - data_batch=[x for x in range(len(test_data))] - if type(test_labels)==list: - labels_batch=[x for x in range(len(test_labels))] - if batch!=None: - total_loss=0 - total_acc=0 - if self.test_dataset!=None: - for data_batch,labels_batch in self.test_dataset: - if self.process_thread==None or t==None: - output=self.nn.fp(data_batch) - else: - output=self.nn.fp(data_batch,t) - batch_loss=self.nn.loss(output,labels_batch) - total_loss+=batch_loss - try: - if self.nn.accuracy!=None: - batch_acc=self.nn.accuracy(output,labels_batch) - total_acc+=batch_acc - except AttributeError: - pass - else: - total_loss=0 - total_acc=0 - if type(test_data)==list: - batches=int((test_data[0].shape[0]-test_data[0].shape[0]%batch)/batch) - shape0=test_data[0].shape[0] - else: - batches=int((test_data.shape[0]-test_data.shape[0]%batch)/batch) - shape0=test_data.shape[0] - for j in range(batches): - index1=j*batch - index2=(j+1)*batch - if type(test_data)==list: - for i in range(len(test_data)): - data_batch[i]=test_data[i][index1:index2] - else: - data_batch=test_data[index1:index2] - if type(test_labels)==list: - for i in range(len(test_labels)): - labels_batch[i]=test_labels[i][index1:index2] - else: - labels_batch=test_labels[index1:index2] - if self.process_thread==None or t==None: - output=self.nn.fp(data_batch) - else: - output=self.nn.fp(data_batch,t) - batch_loss=self.nn.loss(output,labels_batch) - total_loss+=batch_loss - try: - if self.nn.accuracy!=None: - batch_acc=self.nn.accuracy(output,labels_batch) - total_acc+=batch_acc - except AttributeError: - pass - if shape0%batch!=0: - batches+=1 - index1=batches*batch - index2=batch-(shape0-batches*batch) - try: - if type(test_data)==list: - for i in range(len(test_data)): - data_batch[i]=self.platform.concat([test_data[i][index1:],test_data[i][:index2]],0) - else: - data_batch=self.platform.concat([test_data[index1:],test_data[:index2]],0) - if type(self.test_labels)==list: - for i in range(len(test_labels)): - labels_batch[i]=self.platform.concat([test_labels[i][index1:],test_labels[i][:index2]],0) - else: - labels_batch=self.platform.concat([test_labels[index1:],test_labels[:index2]],0) - except: - if type(test_data)==list: - for i in range(len(test_data)): - data_batch[i]=self.platform.concat([test_data[i][index1:],test_data[i][:index2]],0) - else: - data_batch=self.platform.concat([test_data[index1:],test_data[:index2]],0) - if type(self.test_labels)==list: - for i in range(len(test_labels)): - labels_batch[i]=self.platform.concat([test_labels[i][index1:],test_labels[i][:index2]],0) - else: - labels_batch=self.platform.concat([test_labels[index1:],test_labels[:index2]],0) - if self.process_thread==None or t==None: - output=self.nn.fp(data_batch) - else: - output=self.nn.fp(data_batch,t) - batch_loss=self.nn.loss(output,labels_batch) - total_loss+=batch_loss - try: - if self.nn.accuracy!=None: - batch_acc=self.nn.accuracy(output,labels_batch) - total_acc+=batch_acc - except AttributeError: - pass - test_loss=total_loss.numpy()/batches - try: - if self.nn.accuracy!=None: - test_acc=total_acc.numpy()/batches - except AttributeError: - pass - else: - if self.process_thread==None or t==None: - output=self.nn.fp(test_data) - else: - output=self.nn.fp(test_data,t) - test_loss=self.nn.loss(output,test_labels) - test_loss=test_loss.numpy() - try: - if self.nn.accuracy!=None: - test_acc=self.nn.accuracy(output,test_labels) - test_acc=test_acc.numpy() - except AttributeError: - pass - try: - if self.nn.accuracy!=None: - return test_loss,test_acc - except AttributeError: - return test_loss,None - - - def visualize_train(self): - print() - plt.figure(1) - plt.plot(np.arange(self.total_epoch),self.train_loss_list) - plt.title('train loss') - plt.xlabel('epoch') - plt.ylabel('loss') - print('train loss:{0:.6f}'.format(self.train_loss)) - try: - if self.nn.accuracy!=None: - plt.figure(2) - plt.plot(np.arange(self.total_epoch),self.train_acc_list) - plt.title('train acc') - plt.xlabel('epoch') - plt.ylabel('acc') - if self.acc_flag=='%': - print('train acc:{0:.1f}'.format(self.train_acc*100)) - else: - print('train acc:{0:.6f}'.format(self.train_acc)) - except AttributeError: - pass - return - - - def visualize_test(self): - print() - plt.figure(1) - plt.plot(np.arange(self.total_epoch),self.test_loss_list) - plt.title('test loss') - plt.xlabel('epoch') - plt.ylabel('loss') - print('test loss:{0:.6f}'.format(self.test_loss)) - try: - if self.nn.accuracy!=None: - plt.figure(2) - plt.plot(np.arange(self.total_epoch),self.test_acc_list) - plt.title('test acc') - plt.xlabel('epoch') - plt.ylabel('acc') - if self.acc_flag=='%': - print('test acc:{0:.1f}'.format(self.test_acc*100)) - else: - print('test acc:{0:.6f}'.format(self.test_acc)) - except AttributeError: - pass - return - - - def comparison(self): - print() - plt.figure(1) - plt.plot(np.arange(self.total_epoch),self.train_loss_list,'b-',label='train loss') - if self.test_flag==True: - plt.plot(np.arange(self.total_epoch),self.test_loss_list,'r-',label='test loss') - plt.xlabel('epoch') - plt.ylabel('loss') - print('train loss:{0}'.format(self.train_loss)) - plt.legend() - try: - if self.nn.accuracy!=None: - plt.figure(2) - plt.plot(np.arange(self.total_epoch),self.train_acc_list,'b-',label='train acc') - if self.test_flag==True: - plt.plot(np.arange(self.total_epoch),self.test_acc_list,'r-',label='test acc') - plt.xlabel('epoch') - plt.ylabel('acc') - plt.legend() - if self.acc_flag=='%': - print('train acc:{0:.1f}'.format(self.train_acc*100)) - else: - print('train acc:{0:.6f}'.format(self.train_acc)) - except AttributeError: - pass - if self.test_flag==True: - print() - print('-------------------------------------') - print() - print('test loss:{0:.6f}'.format(self.test_loss)) - if self.acc_flag=='%': - print('test acc:{0:.1f}'.format(self.test_acc*100)) - else: - print('test acc:{0:.6f}'.format(self.test_acc)) - return From 282943a7b904383c97901f8c77aec24e48888991 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 29 Jun 2023 13:57:22 +0800 Subject: [PATCH 012/337] Update kernel_reduced.py --- reduced kernel/RL/nspn/kernel_reduced.py | 1126 ++++++++++++---------- 1 file changed, 597 insertions(+), 529 deletions(-) diff --git a/reduced kernel/RL/nspn/kernel_reduced.py b/reduced kernel/RL/nspn/kernel_reduced.py index 5b52841e..0ddd98a9 100644 --- a/reduced kernel/RL/nspn/kernel_reduced.py +++ b/reduced kernel/RL/nspn/kernel_reduced.py @@ -3,615 +3,683 @@ import numpy as np import matplotlib.pyplot as plt import statistics -import pickle -import os import time -#You can analyze kernel by example. -''' -example: -import nspn.kernel_reduced as k #import kernel -import tensorflow as tf -import DQN as d -dqn=d.DQN(4,128,2) #create neural network object -kernel=k.kernel(dqn) #start kernel -kernel.platform=tf -kernel.action_count=2 -kernel.set_up(epsilon=0.01,pool_size=10000,batch=64,update_step=10) -kernel.train(500) -''' class kernel: def __init__(self,nn=None,save_episode=False): - self.nn=nn - self.platform=None - self.state_pool=None - self.action_pool=None - self.next_state_pool=None - self.reward_pool=None - self.done_pool=None - self.episode_set=[] - self.epsilon=None - self.episode_step=None - self.pool_size=None - self.batch=None - self.update_step=None - self.trial_count=None - self.criterion=None - self.reward_list=[] - self.max_episode_count=None - self.save_episode=save_episode - self.filename='save.dat' - self.loss=None - self.loss_list=[] - self.sc=0 - self.total_episode=0 - self.time=0 + self.nn=nn # the neural network model to be trained + try: + self.nn.km=1 # a flag to indicate that the model is using kernel method + except Exception: + pass + self.platform=None # the platform to use, such as TensorFlow or PyTorch + self.state_pool=None # the pool of states + self.action_pool=None # the pool of actions + self.next_state_pool=None # the pool of next states + self.reward_pool=None # the pool of rewards + self.done_pool=None # the pool of done flags + self.episode_set=[] # the list of episodes + self.epsilon=None # the epsilon value for epsilon-greedy policy + self.episode_step=None # the maximum number of steps per episode + self.pool_size=None # the maximum size of the pool + self.batch=None # the batch size for training + self.update_step=None # the frequency of updating the network parameters + self.trial_count=None # the number of trials to calculate the average reward + self.criterion=None # the criterion for stopping the training + self.reward_list=[] # the list of rewards per episode + self.max_episode_count=None # the maximum number of episodes to save + self.save_episode=save_episode # a flag to indicate whether to save episodes or not + self.loss=None # the loss value for training + self.loss_list=[] # the list of losses per episode + self.sc=0 # a counter for steps + self.total_episode=0 # a counter for episodes + self.time=0 # a timer for training time self.total_time=0 def action_vec(self): if self.epsilon!=None: - self.action_one=np.ones(self.action_count,dtype=np.int8) + self.action_one=np.ones(self.action_count,dtype=np.int8) # a vector of ones for action probabilities + return + + + def init(self): + try: + self.nn.pr.TD=np.array(0) # initialize the TD error for prioritized replay buffer + except Exception as e: + try: + if self.nn.pr!=None: + raise e + except Exception: + pass + self.episode_set=[] + self.state_pool=None + self.action_pool=None + self.next_state_pool=None + self.reward_pool=None + self.done_pool=None + self.reward_list=[] + self.loss=0 + self.loss_list=[] + self.sc=0 + self.total_episode=0 + self.time=0 + self.total_time=0 return def set_up(self,epsilon=None,episode_step=None,pool_size=None,batch=None,update_step=None,trial_count=None,criterion=None): - if epsilon!=None: - self.epsilon=epsilon - if episode_step!=None: - self.episode_step=episode_step - if pool_size!=None: - self.pool_size=pool_size - if batch!=None: - self.batch=batch - if update_step!=None: - self.update_step=update_step - if trial_count!=None: - self.trial_count=trial_count - if criterion!=None: - self.criterion=criterion - self.action_vec() + if epsilon!=None: + self.epsilon=epsilon # set the epsilon value for epsilon-greedy policy + if episode_step!=None: + self.episode_step=episode_step # set the maximum number of steps per episode + if pool_size!=None: + self.pool_size=pool_size # set the maximum size of the pool + if batch!=None: + self.batch=batch # set the batch size for training + if update_step!=None: + self.update_step=update_step # set the frequency of updating the network parameters + if trial_count!=None: + self.trial_count=trial_count # set the number of trials to calculate the average reward + if criterion!=None: + self.criterion=criterion # set the criterion for stopping the training + self.action_vec() # create an action vector with the same length as the number of actions, and fill it with ones to indicate the initial probabilities of each action return def epsilon_greedy_policy(self,s): - action_prob=self.action_one*self.epsilon/len(self.action_one) + action_prob=self.action_one*self.epsilon/len(self.action_one) # initialize the action probabilities with epsilon try: - if self.platform.DType!=None: - best_a=np.argmax(self.nn.nn.fp(s)) - action_prob[best_a]+=1-self.epsilon - except AttributeError: - best_a=self.nn.nn(s).argmax() - action_prob[best_a.numpy()]+=1-self.epsilon - return action_prob + best_a=np.argmax(self.nn.nn.fp(s)) # find the best action according to the network output + action_prob[best_a]+=1-self.epsilon # increase the probability of the best action by 1-epsilon + except Exception as e: + try: + if self.platform.DType!=None: # check if the platform is TensorFlow + raise e + except Exception: + best_a=self.nn.nn(s).argmax() # find the best action according to the network output + action_prob[best_a.numpy()]+=1-self.epsilon # increase the probability of the best action by 1-epsilon + return action_prob # return the action probabilities @function(jit_compile=True) def tf_opt(self,state_batch,action_batch,next_state_batch,reward_batch,done_batch): - with self.platform.GradientTape(persistent=True) as tape: - loss=self.nn.loss(state_batch,action_batch,next_state_batch,reward_batch,done_batch) + with self.platform.GradientTape(persistent=True) as tape: # create a persistent gradient tape to record the gradients + loss=self.nn.loss(state_batch,action_batch,next_state_batch,reward_batch,done_batch) # calculate the loss function try: - gradient=self.nn.gradient(tape,loss) try: - self.nn.opt.apply_gradients(zip(gradient,self.nn.param)) - except AttributeError: - self.nn.opt(gradient) - except AttributeError: - try: - if self.nn.nn!=None: - gradient=tape.gradient(loss,self.nn.param) - self.nn.opt.apply_gradients(zip(gradient,self.nn.param)) - except AttributeError: - actor_gradient=tape.gradient(loss[0],self.nn.param[0]) - critic_gradient=tape.gradient(loss[1],self.nn.param[1]) - self.nn.opt.apply_gradients(zip(actor_gradient,self.nn.param[0])) - self.nn.opt.apply_gradients(zip(critic_gradient,self.nn.param[1])) - return loss + gradient=self.nn.gradient(tape,loss) # calculate the gradient using the custom gradient function + try: + self.nn.opt.apply_gradients(zip(gradient,self.nn.param)) # apply the gradient to update the network parameters + except Exception: + self.nn.opt(gradient) # apply the gradient to update the network parameters + except Exception: + try: + if self.nn.nn!=None: # check if the network is a single network or a pair of networks + gradient=tape.gradient(loss,self.nn.param) # calculate the gradient using the tape function + self.nn.opt.apply_gradients(zip(gradient,self.nn.param)) # apply the gradient to update the network parameters + except Exception: + actor_gradient=tape.gradient(loss[0],self.nn.param[0]) # calculate the gradient for the actor network + critic_gradient=tape.gradient(loss[1],self.nn.param[1]) # calculate the gradient for the critic network + self.nn.opt.apply_gradients(zip(actor_gradient,self.nn.param[0])) # apply the gradient to update the actor network parameters + self.nn.opt.apply_gradients(zip(critic_gradient,self.nn.param[1])) # apply the gradient to update the critic network parameters + except Exception as e: + raise e + return loss # return the loss value def pytorch_opt(self,state_batch,action_batch,next_state_batch,reward_batch,done_batch): - loss=self.nn.loss(state_batch,action_batch,next_state_batch,reward_batch,done_batch) - self.nn.backward(loss) - self.nn.opt() - return loss + loss=self.nn.loss(state_batch,action_batch,next_state_batch,reward_batch,done_batch) # calculate the loss function + self.nn.backward(loss) # calculate and accumulate the gradients using the custom backward function + self.nn.opt() # apply the gradients to update the network parameters using the custom opt function + return loss # return the loss value def opt(self,state_batch,action_batch,next_state_batch,reward_batch,done_batch): try: - if self.platform.DType!=None: - loss=self.tf_opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch) - except AttributeError: - loss=self.pytorch_opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch) - return loss - - - def opt_ol(self,state,action,next_state,reward,done): - try: - if self.platform.DType!=None: - loss=self.tf_opt(state,action,next_state,reward,done) - except AttributeError: - loss=self.pytorch_opt(state,action,next_state,reward,done) - return loss + if self.platform.DType!=None: # check if the platform is TensorFlow + loss=self.tf_opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch) # use TensorFlow optimization method + except Exception: + loss=self.pytorch_opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch) # use PyTorch optimization method + return loss # return the loss value def pool(self,s,a,next_s,r,done): - if type(self.state_pool)!=np.ndarray and self.state_pool==None: - self.state_pool=s - if type(a)==int: - a=np.array(a) - self.action_pool=np.expand_dims(a,axis=0) + if type(self.state_pool)!=np.ndarray and self.state_pool==None: # check if the pool is empty or not + self.state_pool=s # initialize the state pool with s + if type(a)==int: # check if a is an integer or a vector + if type(self.nn.param[0])!=list: + a=np.array(a,self.nn.param[0].dtype.name) # convert a to an array with dtype matching nn.param[0] + else: + a=np.array(a,self.nn.param[0][0].dtype.name) # convert a to an array with dtype matching nn.param[0][0] + self.action_pool=np.expand_dims(a,axis=0) # initialize the action pool with a and add an extra dimension for batch size else: - self.action_pool=a - self.next_state_pool=np.expand_dims(next_s,axis=0) - self.reward_pool=np.expand_dims(r,axis=0) - self.done_pool=np.expand_dims(done,axis=0) + if type(self.nn.param[0])!=list: + a=a.astype(self.nn.param[0].dtype.name) # convert a to an array with dtype matching nn.param[0] + else: + a=a.astype(self.nn.param[0][0].dtype.name) # convert a to an array with dtype matching nn.param[0][0] + self.action_poolself.action_pool=a # initialize the action pool with a + self.next_state_pool=np.expand_dims(next_s,axis=0) # initialize the next state pool with next_s and add an extra dimension for batch size + self.reward_pool=np.expand_dims(r,axis=0) # initialize the reward pool with r and add an extra dimension for batch size + self.done_pool=np.expand_dims(done,axis=0) # initialize the done pool with done and add an extra dimension for batch size else: - self.state_pool=np.concatenate((self.state_pool,s),0) - if type(a)==int: - a=np.array(a) - self.action_pool=np.concatenate((self.action_pool,np.expand_dims(a,axis=0)),0) + self.state_pool=np.concatenate((self.state_pool,s),0) # append s to the state pool along the first axis + if type(a)==int: # check if a is an integer or a vector + if type(self.nn.param[0])!=list: + a=np.array(a,self.nn.param[0].dtype.name) # convert a to an array with dtype matching nn.param[0] + else: + a=np.array(a,self.nn.param[0][0].dtype.name) # convert a to an array with dtype matching nn.param[0][0] + self.action_pool=np.concatenate((self.action_pool,np.expand_dims(a,axis=0)),0) # append a to the action pool along the first axis and add an extra dimension for batch size else: - self.action_pool=np.concatenate((self.action_pool,a),0) - self.next_state_pool=np.concatenate((self.next_state_pool,np.expand_dims(next_s,axis=0)),0) - self.reward_pool=np.concatenate((self.reward_pool,np.expand_dims(r,axis=0)),0) - self.done_pool=np.concatenate((self.done_pool,np.expand_dims(done,axis=0)),0) - if len(self.state_pool)>self.pool_size: - self.state_pool=self.state_pool[1:] - self.action_pool=self.action_pool[1:] - self.next_state_pool=self.next_state_pool[1:] - self.reward_pool=self.reward_pool[1:] - self.done_pool=self.done_pool[1:] + if type(self.nn.param[0])!=list: + a=a.astype(self.nn.param[0].dtype.name) # convert a to an array with dtype matching nn.param[0] + else: + a=a.astype(self.nn.param[0][0].dtype.name) # convert a to an array with dtype matching nn.param[0][0] + self.action_pool=np.concatenate((self.action_pool,a),0) # append a to the action pool along the first axis + self.next_state_pool=np.concatenate((self.next_state_pool,np.expand_dims(next_s,axis=0)),0) # append next_s to the next state pool along the first axis and add an extra dimension for batch size + self.reward_pool=np.concatenate((self.reward_pool,np.expand_dims(r,axis=0)),0) # append r to the reward pool along the first axis and add an extra dimension for batch size + self.done_pool=np.concatenate((self.done_pool,np.expand_dims(done,axis=0)),0) # append done to the done pool along the first axis and add an extra dimension for batch size + if len(self.state_pool)>self.pool_size: # check if the pool size exceeds the maximum size + self.state_pool=self.state_pool[1:] # remove the oldest state from the state pool + self.action_pool=self.action_pool[1:] # remove the oldest action from the action pool + self.next_state_pool=self.next_state_pool[1:] # remove the oldest next state from the next state pool + self.reward_pool=self.reward_pool[1:] # remove the oldest reward from the reward pool + self.done_pool=self.done_pool[1:] # remove the oldest done flag from the done pool return def _train(self): - if len(self.state_pool)self.pool_size: # check if the pool size exceeds the maximum size + TD=np.array(0) # create a zero TD error value + self.nn.pr.TD=np.append(TD,self.nn.pr.TD[2:]) # remove the oldest TD error value from the TD error list and append the zero TD error value + except Exception as e: + try: + if self.nn.pr!=None: # check if the prioritized replay buffer is defined or not + raise e # raise the exception if it is defined + except Exception: + pass + self.reward=r+self.reward # accumulate the reward value over steps + loss=self._train() # train the network using the pool data and get the loss value + self.sc+=1 # increase the step counter by one + if done: # check if done flag is True or not + if self.save_episode==True: # check if save episode flag is True or not + episode=[s,a,next_s,r] # create a list of data for this step + self.reward_list.append(self.reward) # append the reward value to the reward list + return loss,episode,done # return the loss value, episode data, and done flag + elif self.save_episode==True: # check if save episode flag is True or not + episode=[s,a,next_s,r] # create a list of data for this step + s=next_s # update s as next_s for next step + else: + for _ in range(self.episode_step): # loop over the maximum number of steps per episode + try: + try: + if self.platform.DType!=None: # check if the platform is TensorFlow + s=np.expand_dims(s,axis=0) # add an extra dimension for batch size to s + if self.epsilon==None: # check if the epsilon value is defined or not + self.epsilon=self.nn.epsilon(self.sc) # calculate the epsilon value using the custom epsilon function + action_prob=self.epsilon_greedy_policy(s) # get the action probabilities using the epsilon-greedy policy function + a=np.random.choice(self.action_count,p=action_prob) # choose an action randomly according to the action probabilities + except Exception: + s=np.expand_dims(s,axis=0) # add an extra dimension for batch size to s + s=self.platform.tensor(s,dtype=self.platform.float).to(self.nn.device) # convert s to a tensor with dtype matching platform.float and move it to the device + if self.epsilon==None: # check if the epsilon value is defined or not + self.epsilon=self.nn.epsilon(self.sc) # calculate the epsilon value using the custom epsilon function + action_prob=self.epsilon_greedy_policy(s) # get the action probabilities using the epsilon-greedy policy function + a=np.random.choice(self.action_count,p=action_prob) # choose an action randomly according to the action probabilities + next_s,r,done=self.nn.env(a) # get the next state, reward, and done flag from the environment using the custom env function and the chosen action + try: + if self.platform.DType!=None: # check if the platform is TensorFlow + if type(self.nn.param[0])!=list: + next_s=np.array(next_s,self.nn.param[0].dtype.name) # convert next_s to an array with dtype matching nn.param[0] + r=np.array(r,self.nn.param[0].dtype.name) # convert r to an array with dtype matching nn.param[0] + done=np.array(done,self.nn.param[0].dtype.name) # convert done to an array with dtype matching nn.param[0] + else: + next_s=np.array(next_s,self.nn.param[0][0].dtype.name) # convert next_s to an array with dtype matching nn.param[0][0] + r=np.array(r,self.nn.param[0][0].dtype.name) # convert r to an array with dtype matching nn.param[0][0] + done=np.array(done,self.nn.param[0][0].dtype.name) # convert done to an array with dtype matching nn.param[0][0] + except Exception: + pass + except Exception as e: + try: + if self.nn.nn!=None: # check if the network is a single network or a pair of networks + raise e # raise the exception if it is a single network + except Exception: try: - if self.platform.DType!=None: - s=np.expand_dims(s,axis=0) - a=(self.nn.actor.fp(s)+self.nn.noise()).numpy() - except AttributeError: - s=np.expand_dims(s,axis=0) - s=self.platform.tensor(s,dtype=self.platform.float).to(self.nn.device) - a=(self.nn.actor(s)+self.nn.noise()).detach().numpy() - next_s,r,done=self.nn.env(a) - r=np.array(r,dtype=np.float32) - done=np.array(done,dtype=np.float32) - self.pool(s,a,next_s,r,done) - self.reward=r+self.reward - loss=self._train() - self.sc+=1 - if done: - if self.save_episode==True: - episode=[s,a,next_s,r] - self.reward_list.append(self.reward) - return loss,episode,done - elif self.save_episode==True: - episode=[s,a,next_s,r] - s=next_s - self.reward_list.append(self.reward) - return loss,episode,done + try: + if self.nn.action!=None: # check if the custom action function is defined or not + try: + if self.platform.DType!=None: # check if the platform is TensorFlow + s=np.expand_dims(s,axis=0) # add an extra dimension for batch size to s + if self.epsilon==None: # check if the epsilon value is defined or not + self.epsilon=self.nn.epsilon(self.sc) # calculate the epsilon value using the custom epsilon function + try: + if self.nn.discriminator!=None: # check if the custom discriminator function is defined or not + a=self.nn.action(s) # get the action using the custom action function + reward=self.nn.discriminator(s,a) # get the reward using the custom discriminator function + s=np.squeeze(s) # remove the extra dimension from s + except Exception: + a=self.nn.action(s).numpy() # get the action using the custom action function and convert it to numpy array + except Exception: + s=np.expand_dims(s,axis=0) # add an extra dimension for batch size to s + s=self.platform.tensor(s,dtype=self.platform.float).to(self.nn.device) # convert s to a tensor with dtype matching platform.float and move it to the device + if self.epsilon==None: # check if the epsilon value is defined or not + self.epsilon=self.nn.epsilon(self.sc) # calculate the epsilon value using the custom epsilon function + try: + if self.nn.discriminator!=None: # check if the custom discriminator function is defined or not + a=self.nn.action(s) # get the action using thecustom action function + reward=self.nn.discriminator(s,a) # get the reward using the custom discriminator function + s=np.squeeze(s) # remove the extra dimension from s + except Exception: + a=self.nn.action(s).detach().numpy() # get the action using the custom action function and convert it to numpy array + except Exception: + try: + if self.platform.DType!=None: # check if the platform is TensorFlow + s=np.expand_dims(s,axis=0) # add an extra dimension for batch size to s + a=(self.nn.actor.fp(s)+self.nn.noise()).numpy() # get the action using the actor network and add some noise and convert it to numpy array + except Exception: + s=np.expand_dims(s,axis=0) # add an extra dimension for batch size to s + s=self.platform.tensor(s,dtype=self.platform.float).to(self.nn.device) # convert s to a tensor with dtype matching platform.float and move it to the device + a=(self.nn.actor(s)+self.nn.noise()).detach().numpy() # get the action using the actor network and add some noise and convert it to numpy array + except Exception as e: + raise e # raise the exception if any error occurs + next_s,r,done=self.nn.env(a) # get the next state, reward, and done flag from the environment using the custom env function and the chosen action + try: + if self.platform.DType!=None: # check if the platform is TensorFlow + if type(self.nn.param[0])!=list: + next_s=np.array(next_s,self.nn.param[0].dtype.name) # convert next_s to an array with dtype matching nn.param[0] + r=np.array(r,self.nn.param[0].dtype.name) # convert r to an array with dtype matching nn.param[0] + done=np.array(done,self.nn.param[0].dtype.name) # convert done to an array with dtype matching nn.param[0] + else: + next_s=np.array(next_s,self.nn.param[0][0].dtype.name) # convert next_s to an array with dtype matching nn.param[0][0] + r=np.array(r,self.nn.param[0][0].dtype.name) # convert r to an array with dtype matching nn.param[0][0] + done=np.array(done,self.nn.param[0][0].dtype.name) # convert done to an array with dtype matching nn.param[0][0] + except Exception: + pass + try: + self.nn.pool(self.state_pool,self.action_pool,self.next_state_pool,self.reward_pool,self.done_pool,[s,a,next_s,reward,done]) # add the data to the pool using the custom pool function + except Exception as e: + try: + if self.nn.pool!=None: # check if the custom pool function is defined or not + raise e # raise the exception if it is defined + except Exception: + self.pool(s,a,next_s,r,done) # add the data to the pool using the default pool function + try: + self.nn.pr.TD=np.append(self.nn.pr.TD,self.nn.initial_TD) # append the initial TD error value to the TD error list for prioritized replay buffer + if len(self.state_pool)>self.pool_size: # check if the pool size exceeds the maximum size + TD=np.array(0) # create a zero TD error value + self.nn.pr.TD=np.append(TD,self.nn.pr.TD[2:]) # remove the oldest TD error value from the TD error list and append the zero TD error value + except Exception as e: + try: + if self.nn.pr!=None: # check if the prioritized replay buffer is defined or not + raise e # raise the exception if it is defined + except Exception: + pass + self.reward=r+self.reward # accumulate the reward value over steps + loss=self._train() # train the network using the pool data and get the loss value + self.sc+=1 # increase the step counter by one + if done: # check if done flag is True or not + if self.save_episode==True: # check if save episode flag is True or not + episode=[s,a,next_s,r] # create a list of data for this step + self.reward_list.append(self.reward) # append the reward value to the reward list + return loss,episode,done # return the loss value, episode data, and done flag + elif self.save_episode==True: # check if save episode flag is True or not + episode=[s,a,next_s,r] # create a list of data for this step + s=next_s # update s as next_s for next step + self.reward_list.append(self.reward) # append the reward value to the reward list + return loss,episode,done # return the loss value, episode data, and done flag def train(self,episode_count,save=None,one=True,p=None,s=None): - avg_reward=None - if p==None: - self.p=9 + avg_reward=None # initialize the average reward value as None + if p==None: # check if p is defined or not + self.p=9 # set p as 9 by default else: - self.p=p-1 - if s==None: - self.s=1 - self.file_list=None + self.p=p-1 # decrease p by one + if s==None: # check if s is defined or not + self.s=1 # set s as 1 by default + self.file_list=None # set file_list as None by default else: - self.s=s-1 - self.file_list=[] - if episode_count!=None: - for i in range(episode_count): - t1=time.time() - loss,episode,done=self.train_() - if self.trial_count!=None: - if len(self.reward_list)>=self.trial_count: - avg_reward=statistics.mean(self.reward_list[-self.trial_count:]) - if self.criterion!=None and avg_reward>=self.criterion: - t2=time.time() - self.total_time+=(t2-t1) - self._time=self.total_time-int(self.total_time) - if self._time<0.5: - self.total_time=int(self.total_time) + self.s=s-1 # decrease s by one + self.file_list=[] # initialize an empty list for file_list + if episode_count!=None: # check if episode_count is defined or not + for i in range(episode_count): # loop over each episode + t1=time.time() # record the start time of the episode + loss,episode,done=self.train_() # train the network for one episode and get the loss value, episode data, and done flag + if self.trial_count!=None: # check if trial_count is defined or not + if len(self.reward_list)>=self.trial_count: # check if the reward list has enough values for trial_count + avg_reward=statistics.mean(self.reward_list[-self.trial_count:]) # calculate the average reward value using the last trial_count values in the reward list + if self.criterion!=None and avg_reward>=self.criterion: # check if criterion is defined or not and if the average reward value meets the criterion or not + t2=time.time() # record the end time of the episode + self.total_time+=(t2-t1) # accumulate the total time over episodes + self._time=self.total_time-int(self.total_time) # get the decimal part of the total time + if self._time<0.5: + self.total_time=int(self.total_time) # round down the total time if the decimal part is less than 0.5 else: - self.total_time=int(self.total_time)+1 - print('episode:{0}'.format(self.total_episode)) - print('last loss:{0:.6f}'.format(loss)) - print('average reward:{0}'.format(avg_reward)) + self.total_time=int(self.total_time)+1 # round up the total time if the decimal part is greater than or equal to 0.5 + print('episode:{0}'.format(self.total_episode)) # print the total number of episodes + print('last loss:{0:.6f}'.format(loss)) # print the last loss value + print('average reward:{0}'.format(avg_reward)) # print the average reward value print() - print('time:{0}s'.format(self.total_time)) - return - self.loss=loss - self.loss_list.append(loss) - self.total_episode+=1 - if episode_count%10!=0: - p=episode_count-episode_count%self.p - p=int(p/self.p) - s=episode_count-episode_count%self.s - s=int(s/self.s) - else: - p=episode_count/(self.p+1) - p=int(p) - s=episode_count/(self.s+1) - s=int(s) - if p==0: - p=1 - if s==0: - s=1 - if i%p==0: - if len(self.state_pool)>=self.batch: - print('episode:{0} loss:{1:.6f}'.format(i+1,loss)) - if avg_reward!=None: - print('episode:{0} average reward:{1}'.format(i+1,avg_reward)) + print('time:{0}s'.format(self.total_time)) # print the total time in seconds + return # stop training and return + self.loss=loss # update loss as the last loss value + self.loss_list.append(loss) # append loss to the loss list + self.total_episode+=1 # increase the total episode counter by one + if episode_count%10!=0: + p=episode_count-episode_count%self.p + p=int(p/self.p) + s=episode_count-episode_count%self.s + s=int(s/self.s) + else: + p=episode_count/(self.p+1) + p=int(p) + s=episode_count/(self.s+1) + s=int(s) + if p==0: + p=1 + if s==0: + s=1 + if i%p==0: # check if i is a multiple of p or not + if len(self.state_pool)>=self.batch: # check if the pool size is larger than or equal to the batch size or not + print('episode:{0} loss:{1:.6f}'.format(i+1,loss)) # print the episode number and loss value + if avg_reward!=None: # check if avg_reward is None or not + print('episode:{0} average reward:{1}'.format(i+1,avg_reward)) # print the episode number and average reward value else: - print('episode:{0} reward:{1}'.format(i+1,self.reward)) + print('episode:{0} reward:{1}'.format(i+1,self.reward)) # print the episode number and reward value print() - if save!=None and i%s==0: - self.save(self.total_episode,one) - if self.save_episode==True: - if done: - episode.append('done') - self.episode_set.append(episode) - if self.max_episode_count!=None and len(self.episode_set)>=self.max_episode_count: - self.save_episode=False + if save!=None and i%s==0: # check if save is None or not and if i is a multiple of s orif save!=None and i%s==0: # check if save is None or not and if i is a multiple of s or not + self.save(self.total_episode,one) # save the network parameters and training data using the custom save function and the total episode number and one flag + if self.save_episode==True: # check if save episode flag is True or not + if done: # check if done flag is True or not + episode.append('done') # append 'done' to the episode data list + self.episode_set.append(episode) # append the episode data list to the episode set list + if self.max_episode_count!=None and len(self.episode_set)>=self.max_episode_count: # check if max episode count is defined or not and if the episode set size reaches the max episode count or not + self.save_episode=False # set the save episode flag as False to stop saving episodes try: - self.nn.ec+=1 - except AttributeError: - pass - t2=time.time() - self.time+=(t2-t1) - else: - i=0 - while True: - t1=time.time() - loss,episode,done=self.train_() - if self.trial_count!=None: - if len(self.reward_list)==self.trial_count: - avg_reward=statistics.mean(self.reward_list[-self.trial_count:]) - if avg_reward>=self.criterion: - t2=time.time() - self.total_time+=(t2-t1) - self._time=self.total_time-int(self.total_time) - if self._time<0.5: - self.total_time=int(self.total_time) + try: + self.nn.ec.assign_add(1) # increase the episode counter by one using TensorFlow assign_add method + except Exception: + self.nn.ec+=1 # increase the episode counter by one using Python addition operator + except Exception: + pass + t2=time.time() # record the end time of the episode + self.time+=(t2-t1) # accumulate the time over episodes + else: + i=0 # initialize the episode index as zero + while True: # loop until an exception occurs or the criterion is met + t1=time.time() # record the start time of the episode + loss,episode,done=self.train_() # train the network for one episode and get the loss value, episode data, and done flag + if self.trial_count!=None: # check if trial_count is defined or not + if len(self.reward_list)==self.trial_count: # check if the reward list has enough values for trial_count + avg_reward=statistics.mean(self.reward_list[-self.trial_count:]) # calculate the average reward value using the last trial_count values in the reward list + if avg_reward>=self.criterion: # check if the average reward value meets the criterion or not + t2=time.time() # record the end time of the episode + self.total_time+=(t2-t1) # accumulate the total time over episodes + self._time=self.total_time-int(self.total_time) # get the decimal part of the total time + if self._time<0.5: + self.total_time=int(self.total_time) # round down the total time if the decimal part is less than 0.5 else: - self.total_time=int(self.total_time)+1 - print('episode:{0}'.format(self.total_episode)) - print('last loss:{0:.6f}'.format(loss)) - print('average reward:{0}'.format(avg_reward)) + self.total_time=int(self.total_time)+1 # round up the total time if the decimal part is greater than or equal to 0.5 + print('episode:{0}'.format(self.total_episode)) # print the total number of episodes + print('last loss:{0:.6f}'.format(loss)) # print the last loss value + print('average reward:{0}'.format(avg_reward)) # print the average reward value print() - print('time:{0}s'.format(self.total_time)) - return - self.loss=loss - self.loss_list.append(loss) - i+=1 - self.total_episode+=1 - if episode_count%10!=0: - p=episode_count-episode_count%self.p - p=int(p/self.p) - s=episode_count-episode_count%self.s - s=int(s/self.s) - else: - p=episode_count/(self.p+1) - p=int(p) - s=episode_count/(self.s+1) - s=int(s) - if p==0: - p=1 - if s==0: - s=1 - if i%p==0: - if len(self.state_pool)>=self.batch: - print('episode:{0} loss:{1:.6f}'.format(i+1,loss)) - if avg_reward!=None: - print('episode:{0} average reward:{1}'.format(i+1,avg_reward)) - else: - print('episode:{0} reward:{1}'.format(i+1,self.reward)) + print('time:{0}s'.format(self.total_time)) # print the total time in seconds + return # stop training and return + self.loss=loss # update loss as the last loss value + self.loss_list.append(loss) # append loss to the loss list + i+=1 # increase the episode index by one + self.total_episode+=1 # increase the total episode counter by one + if episode_count%10!=0: + p=episode_count-episode_count%self.p + p=int(p/self.p) + s=episode_count-episode_count%self.s + s=int(s/self.s) + else: + p=episode_count/(self.p+1) + p=int(p) + s=episode_count/(self.s+1) + s=int(s) + if p==0: + p=1 + if s==0: + s=1 + if i%p==0: # check if i is a multiple of p or not + if len(self.state_pool)>=self.batch: # check if the pool size is larger than or equal to the batch size or not + print('episode:{0} loss:{1:.6f}'.format(i+1,loss)) # print the episode number and loss value + if avg_reward!=None: # check if avg_reward is None or not + print('episode:{0} average reward:{1}'.format(i+1,avg_reward)) # print the episode number and average reward valueelse: + else: + print('episode:{0} reward:{1}'.format(i+1,self.reward)) # print the episode number and reward value print() - if save!=None and i%s==0: - self.save(self.total_episode,one) - if self.save_episode==True: - if done: - episode.append('done') - self.episode_set.append(episode) - if self.max_episode_count!=None and len(self.episode_set)>=self.max_episode_count: - self.save_episode=False + if save!=None and i%s==0: # check if save is None or not and if i is a multiple of s or not + self.save(self.total_episode,one) # save the network parameters and training data using the custom save function and the total episode number and one flag + if self.save_episode==True: # check if save episode flag is True or not + if done: # check if done flag is True or not + episode.append('done') # append 'done' to the episode data list + self.episode_set.append(episode) # append the episode data list to the episode set list + if self.max_episode_count!=None and len(self.episode_set)>=self.max_episode_count: # check if max episode count is defined or not and if the episode set size reaches the max episode count or not + self.save_episode=False # set the save episode flag as False to stop saving episodes try: - self.nn.ec+=1 - except AttributeError: - pass - t2=time.time() - self.time+=(t2-t1) - self._time=self.time-int(self.time) - if self._time<0.5: - self.total_time=int(self.time) + try: + self.nn.ec.assign_add(1) # increase the episode counter by one using TensorFlow assign_add method + except Exception: + self.nn.ec+=1 # increase the episode counter by one using Python addition operator + except Exception: + pass + t2=time.time() # record the end time of the episode + self.time+=(t2-t1) # accumulate the time over episodes + self._time=self.time-int(self.time) # get the decimal part of the time + if self._time<0.5: + self.total_time=int(self.time) # round down the time if the decimal part is less than 0.5 else: - self.total_time=int(self.time)+1 - self.total_time+=self.time - print('last loss:{0:.6f}'.format(loss)) - print('last reward:{0}'.format(self.reward)) + self.total_time=int(self.time)+1 # round up the time if the decimal part is greater than or equal to 0.5 + self.total_time+=self.time # add the time to the total time + print('last loss:{0:.6f}'.format(loss)) # print the last loss value + print('last reward:{0}'.format(self.reward)) # print the last reward value print() - print('time:{0}s'.format(self.time)) - return - - - def train_ol(self): - while True: - if self.stop_flag==True: - return - if self.save_flag==True: - self.save() - self.suspend_func() - data=self.nn.ol() - if data=='stop': - return - elif data=='suspend': - while True: - self.nn.suspend=True - while True: - if self.nn.suspend==False: - break - continue - loss=self.opt_ol(data[0],data[1],data[2],data[3],data[4]) - loss=loss.numpy() - self.nn.train_loss_list.append(loss) - if len(self.nn.train_acc_list)==self.nn.max_length: - del self.nn.train_acc_list[0] - try: - self.nn.c+=1 - except AttributeError: - pass - return - + print('time:{0}s'.format(self.time)) # print the time in seconds + return # return def visualize_reward(self): print() - plt.figure(1) - plt.plot(np.arange(self.total_episode),self.reward_list) - plt.xlabel('episode') - plt.ylabel('reward') - print('reward:{0:.6f}'.format(self.reward_list[-1])) + plt.figure(1) # create a figure object with number 1 + plt.plot(np.arange(self.total_episode),self.reward_list) # plot the reward list against the episode number + plt.xlabel('episode') # set the x-axis label as 'episode' + plt.ylabel('reward') # set the y-axis label as 'reward' + print('reward:{0:.6f}'.format(self.reward_list[-1])) # print the last reward value return def visualize_train(self): print() - plt.figure(1) - plt.plot(np.arange(self.total_episode),self.loss_list) - plt.title('train loss') - plt.xlabel('episode') - plt.ylabel('loss') - print('loss:{0:.6f}'.format(self.loss_list[-1])) - return - - - def save_e(self): - episode_file=open('episode.dat','wb') - pickle.dump(self.episode_set,episode_file) - episode_file.close() - return - - - def save(self,i=None,one=True): - if one==True: - output_file=open(self.filename,'wb') - if self.save_episode==True: - episode_file=open('episode.dat','wb') - pickle.dump(self.episode_set,episode_file) - episode_file.close() - else: - filename=self.filename.replace(self.filename[self.filename.find('.'):],'-{0}.dat'.format(i)) - output_file=open(filename,'wb') - self.file_list.append([filename]) - if self.save_episode==True: - episode_file=open('episode-{0}.dat'.format(i),'wb') - pickle.dump(self.episode_set,episode_file) - episode_file.close() - if self.save_episode==True: - self.file_list.append([filename,'episode-{0}.dat']) - if len(self.file_list)>self.s+1: - os.remove(self.file_list[0][0]) - os.remove(self.file_list[0][1]) - del self.file_list[0] - else: - self.file_list.append([filename]) - if len(self.file_list)>self.s+1: - os.remove(self.file_list[0][0]) - del self.file_list[0] - try: - if self.platform.DType!=None: - try: - pickle.dump(self.nn,output_file) - except: - opt=self.nn.opt - self.nn.opt=None - pickle.dump(self.nn,output_file) - self.nn.opt=opt - except AttributeError: - pass - try: - pickle.dump(self.platform.keras.optimizers.serialize(opt),output_file) - except: - pickle.dump(self.nn.serialize(),output_file) - else: - pickle.dump(None,output_file) - pickle.dump(self.epsilon,output_file) - pickle.dump(self.episode_step,output_file) - pickle.dump(self.pool_size,output_file) - pickle.dump(self.batch,output_file) - pickle.dump(self.update_step,output_file) - pickle.dump(self.max_episode_count,output_file) - pickle.dump(self.save_episode,output_file) - pickle.dump(self.reward_list,output_file) - pickle.dump(self.loss,output_file) - pickle.dump(self.loss_list,output_file) - pickle.dump(self.sc,output_file) - pickle.dump(self.total_episode,output_file) - pickle.dump(self.total_time,output_file) - output_file.close() - return - - - def restore(self,s_path,e_path=None): - input_file=open(s_path,'rb') - if e_path!=None: - episode_file=open(e_path,'rb') - self.episode_set=pickle.load(episode_file) - episode_file.close() - self.nn=pickle.load(input_file) - opt_serialized=pickle.load(input_file) - try: - self.nn.opt=self.platform.keras.optimizers.deserialize(opt_serialized) - except: - self.nn.deserialize(opt_serialized) - else: - pass - self.epsilon=pickle.load(input_file) - self.episode_step=pickle.load(input_file) - self.pool_size=pickle.load(input_file) - self.batch=pickle.load(input_file) - self.update_step=pickle.load(input_file) - self.max_episode_count=pickle.load(input_file) - self.save_episode=pickle.load(input_file) - self.reward_list=pickle.load(input_file) - self.loss=pickle.load(input_file) - self.loss_list=pickle.load(input_file) - self.sc=pickle.load(input_file) - self.total_episode=pickle.load(input_file) - self.total_time=pickle.load(input_file) - input_file.close() + plt.figure(1) # create a figure object with number 1 + plt.plot(np.arange(self.total_episode),self.loss_list) # plot the loss list against the episode number + plt.title('train loss') # set the title of the figure as 'train loss' + plt.xlabel('episode') # set the x-axis label as 'episode' + plt.ylabel('loss') # set the y-axis label as 'loss' + print('loss:{0:.6f}'.format(self.loss_list[-1])) # print the last loss value return From 1da4eb327f20f41804ad871622fcf67984cbff8b Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 29 Jun 2023 14:03:10 +0800 Subject: [PATCH 013/337] Update kernel.txt --- Note 7.0 pv documentation/DL/kernel.txt | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Note 7.0 pv documentation/DL/kernel.txt b/Note 7.0 pv documentation/DL/kernel.txt index 2437fb64..90b86969 100644 --- a/Note 7.0 pv documentation/DL/kernel.txt +++ b/Note 7.0 pv documentation/DL/kernel.txt @@ -35,10 +35,12 @@ kernel.train(32,5) #train neural network ----------------------------------------------------------------------------------------- Parallel optimization: You can use parallel optimization to speed up your training,parallel optimization speed up training by multiprocessing. -Note have two types of parallel optimization: -1. parallel forward propagation and optimizing. -2. parallel forward propagation and calculate a gradient and optimizing. -3. parallel forward propagation and calculate multiple gradients and optimizing. +Note have three types of parallel optimization: +1. Perform forward propagation and optimization in parallel. (PO1) + +2. Perform forward propagation, one gradient computation or multiple gradient computations and optimization in parallel. (PO2) + +3. Perform forward propagation, gradient computation and optimization in parallel without locks. (PO3) ---------------------------------------------------------------------------------------- From db36ff9c4c5b572c82f37228ba14b67fd1e73a1d Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Fri, 30 Jun 2023 21:30:03 +0800 Subject: [PATCH 014/337] Update README.md --- README.md | 352 +++++++----------------------------------------------- 1 file changed, 43 insertions(+), 309 deletions(-) diff --git a/README.md b/README.md index a3fc53b5..90fb5e36 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,5 @@ # Stop training and saving when condition is met: ## DL: -**example(Stop training and saving when condition is met.):** ```python import Note.DL.kernel as k #import kernel import tensorflow as tf #import platform @@ -20,7 +19,6 @@ kernel.train(32,5) #train neural network ``` ## RL: -**example(Stop training and saving when condition is met.):** ```python import Note.RL.kernel as k #import kernel import DQN as d @@ -42,29 +40,6 @@ kernel.visualize_reward() https://github.com/NoteDancing/Note-documentation/blob/Note-7.0-pv/Note%207.0%20pv%20documentation/DL/neural%20network/tensorflow/nn_acc.py -**example(parallel test):** -```python -import Note.DL.kernel as k #import kernel -import tensorflow as tf #import platform -import nn_acc as n #import neural network -import threading -mnist=tf.keras.datasets.mnist -(x_train,y_train),(x_test,y_test)=mnist.load_data() -x_train,x_test =x_train/255.0,x_test/255.0 -nn=n.nn() #create neural network object -kernel=k.kernel(nn) #start kernel -kernel.threading=threading -kernel.thread_t=6 #test thread count -kernel.platform=tf #use platform -kernel.data(x_train,y_train,x_test,y_test) #input train data -kernel.train(32,5,32) #train neural network - #batch size:32 - #test batch size:32 - #epoch:5 -kernel.save() #save neural network -``` - -**example(test module):** ```python import nn_acc as n import Note.DL.dl.test as t @@ -88,8 +63,8 @@ loss,acc=test.loss_acc() # Multiprocessing: -## PO2: -**multiprocessing example(PO2):** +## DL: +### PO2: ```python import Note.DL.process.kernel as k #import kernel import tensorflow as tf @@ -111,7 +86,7 @@ kernel.data(x_train,y_train) #input train data manager=Manager() #create manager object kernel.init(manager) #initialize shared data lock=[Lock(),Lock()] -g_lock=[Lock(),Lock(),Lock()] +g_lock=Lock() for p in range(7): Process(target=kernel.train,args=(p,lock,g_lock)).start() kernel.update_nn_param() @@ -144,8 +119,7 @@ for p in range(7): kernel.update_nn_param() kernel.test(x_train,y_train,32) ``` -## Stop training and saving when condition is met: -**multiprocessing example(Stop training and saving when condition is met.):** +### Stop training and saving when condition is met: ```python import Note.DL.process.kernel as k #import kernel import tensorflow as tf @@ -169,11 +143,11 @@ kernel.data(x_train,y_train) #input train data manager=Manager() #create manager object kernel.init(manager) #initialize shared data lock=[Lock(),Lock()] -g_lock=[Lock(),Lock(),Lock()] +g_lock=Lock() for p in range(7): Process(target=kernel.train,args=(p,lock,g_lock)).start() ``` -## PO3： +### PO3： ```python import Note.DL.process.kernel as k #import kernel import tensorflow as tf @@ -225,285 +199,65 @@ kernel.test(x_train,y_train,32) ``` -# Multithreading: -## DL： -**You can download neural network example in this link,and then you can import neural network and train with kernel,link and example code are below.** - -https://github.com/NoteDancing/Note-documentation/blob/Note-7.0-pv/Note%207.0%20pv%20documentation/DL/neural%20network/tensorflow/nn.py - -**multithreading example:** -```python -import Note.DL.kernel as k #import kernel -import tensorflow as tf #import platform -import nn as n #import neural network -import threading -mnist=tf.keras.datasets.mnist -(x_train,y_train),(x_test,y_test)=mnist.load_data() -x_train,x_test =x_train/255.0,x_test/255.0 -nn=n.nn() #create neural network object -kernel=k.kernel(nn) #start kernel -kernel.platform=tf #use platform -kernel.thread=7 #thread count,use 7 threads to train -kernel.epoch_=6 #epoch:6 -kernel.PO=2 #use PO2 -kernel.data(x_train,y_train) #input train data -kernel.lock=[threading.Lock(),threading.Lock(),threading.Lock()] -class thread(threading.Thread): - def run(self): - kernel.train(32) #batch size:32 -for _ in range(7): - _thread=thread() - _thread.start() -for _ in range(7): - _thread.join() -kernel.visualize_train() -``` - -**multithreading example(segment data):** -```python -import Note.DL.kernel as k #import kernel -import tensorflow as tf #import platform -import nn as n #import neural network -import threading -mnist=tf.keras.datasets.mnist -(x_train,y_train),(x_test,y_test)=mnist.load_data() -x_train,x_test =x_train/255.0,x_test/255.0 -nn=n.nn() #create neural network object -kernel=k.kernel(nn) #start kernel -kernel.platform=tf #use platform -kernel.thread=7 #thread count,use 7 threads to train -kernel.data_segment_flag=True -kernel.batches=1875 #batches:1875 -kernel.epoch_=6 #epoch:6 -kernel.PO=2 -kernel.data(x_train,y_train) #input train data -kernel.lock=[threading.Lock(),threading.Lock(),threading.Lock()] -class thread(threading.Thread): - def run(self): - kernel.train(32) #batch size:32 -for _ in range(7): - _thread=thread() - _thread.start() -for _ in range(7): - _thread.join() -kernel.visualize_train() -``` - -**multithreading example:** -```python -import Note.DL.kernel as k #import kernel -import tensorflow as tf #import platform -import nn as n #import neural network -import threading -mnist=tf.keras.datasets.mnist -(x_train,y_train),(x_test,y_test)=mnist.load_data() -x_train,x_test =x_train/255.0,x_test/255.0 -nn=n.nn() #create neural network object -kernel=k.kernel(nn) #start kernel -kernel.platform=tf #use platform -kernel.thread=2 #thread count,use 2 threads to train -kernel.PO=2 #use PO2 -kernel.data(x_train,y_train) #input train data -kernel.lock=[threading.Lock(),threading.Lock(),threading.Lock()] -class thread(threading.Thread): - def run(self): - kernel.train(32,3) #batch size:32 epoch:3 -for _ in range(2): - _thread=thread() - _thread.start() -for _ in range(2): - _thread.join() -kernel.visualize_train() -``` - -### Stop training and saving when condition is met: -**multithreading example(Stop training and saving when condition is met.):** -```python -import Note.DL.kernel as k #import kernel -import tensorflow as tf #import platform -import nn as n #import neural network -import threading -mnist=tf.keras.datasets.mnist -(x_train,y_train),(x_test,y_test)=mnist.load_data() -x_train,x_test =x_train/255.0,x_test/255.0 -nn=n.nn() #create neural network object -kernel=k.kernel(nn) #start kernel -kernel.platform=tf #use platform -kernel.stop=True -kernel.end_loss=0.7 #stop condition -kernel.thread=2 #thread count,use 2 threads to train -kernel.PO=2 -kernel.data(x_train,y_train) #input train data -kernel.lock=[threading.Lock(),threading.Lock(),threading.Lock()] -class thread(threading.Thread): - def run(self): - kernel.train(32,3) #batch size:32 epoch:3 -for _ in range(2): - _thread=thread() - _thread.start() -for _ in range(2): - _thread.join() -kernel.train_loss_list or kernel.train_loss #view training loss -kernel.visualize_train() -``` - -**Gradient Attenuation：** - -**Calculate the attenuation coefficient based on the optimization counter using the attenuation function.** - -**example:https://github.com/NoteDancing/Note-documentation/blob/Note-7.0-pv/Note%207.0%20pv%20documentation/DL/neural%20network/tensorflow/nn_attenuate.py** - ## RL： ### Pool Network: **You can download neural network example in this link,and then you can import neural network and train with kernel,link and example code are below.** -https://github.com/NoteDancing/Note-documentation/blob/Note-7.0-pv/Note%207.0%20pv%20documentation/RL/neural%20network/tensorflow/pool%20net/thread/DQN.py +https://github.com/NoteDancing/Note-documentation/blob/Note-7.0-pv/Note%207.0%20pv%20documentation/RL/neural%20network/tensorflow/pool%20net/DQN.py -**multithreading example:** +#### PO2: ```python -import Note.RL.thread.kernel as k #import kernel +import Note.RL.kernel as k #import kernel import DQN as d -import threading +from multiprocessing import Process,Lock,Manager dqn=d.DQN(4,128,2) #create neural network object kernel=k.kernel(dqn,5) #start kernel,use 5 thread to train +manager=Manager() #create manager object +kernel.init(manager) #initialize shared data kernel.action_count=2 kernel.set_up(epsilon=0.01,pool_size=10000,batch=64,update_step=10) -kernel.PO=1 #use PO1 -kernel.threading=threading -kernel.lock=[threading.Lock(),threading.Lock(),threading.Lock()] -class thread(threading.Thread): - def run(self): - kernel.train(100) -for _ in range(5): - _thread=thread() - _thread.start() -for _ in range(5): - _thread.join() -kernel.visualize_train() -kernel.visualize_reward() +kernel.PO=2 #use PO2 +pool_lock=[Lock(),Lock(),Lock(),Lock(),Lock()] +lock=[Lock(),Lock(),Lock()] +g_lock=Lock() +for p in range(5): + Process(target=kernel.train,args=(p,100,lock,pool_lock,g_lock)).start() ``` -### Stop training and saving when condition is met: -**multithreading example(Stop training and saving when condition is met.):** +#### PO3: ```python import Note.RL.kernel as k #import kernel import DQN as d -import threading +from multiprocessing import Process,Lock,Manager dqn=d.DQN(4,128,2) #create neural network object kernel=k.kernel(dqn,5) #start kernel,use 5 thread to train -kernel.stop=True +manager=Manager() #create manager object +kernel.init(manager) #initialize shared data kernel.action_count=2 -kernel.set_up(epsilon=0.01,pool_size=10000,batch=64,update_step=10,trial_count=10,criterion=200) -kernel.PO=1 -kernel.threading=threading -kernel.lock=[threading.Lock(),threading.Lock(),threading.Lock()] -class thread(threading.Thread): - def run(self): - kernel.train(100) -for _ in range(5): - _thread=thread() - _thread.start() -for _ in range(5): - _thread.join() -kernel.loss_list or kernel.loss #view training loss -kernel.visualize_train() -kernel.reward #view reward -kernel.visualize_reward() -``` - -## PO3: -**multithreading example(PO3):** -```python -import Note.DL.kernel as k #import kernel -import tensorflow as tf #import platform -import nn as n #import neural network -import threading -mnist=tf.keras.datasets.mnist -(x_train,y_train),(x_test,y_test)=mnist.load_data() -x_train,x_test =x_train/255.0,x_test/255.0 -nn=n.nn() #create neural network object -kernel=k.kernel(nn) #start kernel -kernel.platform=tf #use platform -kernel.thread=7 #thread count,use 7 threads to train +kernel.set_up(epsilon=0.01,pool_size=10000,batch=64,update_step=10) kernel.PO=3 #use PO3 -kernel.threading=threading -kernel.max_lock=7 -kernel.data(x_train,y_train) #input train data -kernel.lock=[threading.Lock(),threading.Lock()] -class thread(threading.Thread): - def run(self): - kernel.train(32,1) #batch size:32 epoch:1 -for _ in range(7): - _thread=thread() - _thread.start() -for _ in range(7): - _thread.join() -``` - -## Parallel test: -**You can download neural network example in this link,and then you can import neural network and train with kernel,link and example code are below.** - -https://github.com/NoteDancing/Note-documentation/blob/Note-7.0-pv/Note%207.0%20pv%20documentation/DL/neural%20network/tensorflow/nn_acc.py - -**multithreading example(parallel test):** -```python -import Note.DL.kernel as k #import kernel -import tensorflow as tf #import platform -import nn_acc as n #import neural network -import threading -mnist=tf.keras.datasets.mnist -(x_train,y_train),(x_test,y_test)=mnist.load_data() -x_train,x_test =x_train/255.0,x_test/255.0 -nn=n.nn() #create neural network object -kernel=k.kernel(nn) #start kernel -kernel.platform=tf #use platform -kernel.threading=threading -kernel.thread=7 #thread count,use 7 threads to train -kernel.thread_t=6 #test thread count -kernel.epoch_=6 #epoch:6 -kernel.PO=2 -kernel.data(x_train,y_train,x_test,y_test) #input train data -kernel.lock=[threading.Lock(),threading.Lock(),threading.Lock()] -class thread(threading.Thread): - def run(self): - kernel.train(32,test_batch=32) #batch size:32 -for _ in range(7): - _thread=thread() - _thread.start() -for _ in range(7): - _thread.join() -kernel.visualize_train() +pool_lock=[Lock(),Lock(),Lock(),Lock(),Lock()] +lock=[Lock(),Lock()] +g_lock=Lock() +for p in range(5): + Process(target=kernel.train,args=(p,100,lock,pool_lock)).start() ``` - -## Online training: -**You can download neural network example in this link,and then you can import neural network and train with kernel,link and example code are below.** - -https://github.com/NoteDancing/Note-documentation/blob/Note-7.0-pv/Note%207.0%20pv%20documentation/DL/neural%20network/tensorflow/nn_ol_p.py - -**multithreading example:** ```python -import Note.DL.kernel as k #import kernel -import tensorflow as tf #import platform -import nn_ol_p as n #import neural network -import threading -mnist=tf.keras.datasets.mnist -(x_train,y_train),(x_test,y_test)=mnist.load_data() -x_train,x_test =x_train/255.0,x_test/255.0 -nn=n.nn(x_train,y_train) #create neural network object -kernel=k.kernel(nn) #start kernel -kernel.platform=tf #use platform -kernel.thread=7 #thread count,use 7 thread to train -kernel.PO=2 -kernel.create_t_num(7) #input train data -kernel.lock=[threading.Lock(),threading.Lock(),threading.Lock()] -class thread(threading.Thread): - def run(self): - kernel.train_ol() -for _ in range(7): - _thread=thread() - _thread.start() -for _ in range(7): - _thread.join() +import Note.RL.kernel as k #import kernel +import DQN as d +from multiprocessing import Process,Lock,Manager +dqn=d.DQN(4,128,2) #create neural network object +kernel=k.kernel(dqn,5) #start kernel,use 5 thread to train +manager=Manager() #create manager object +kernel.init(manager) #initialize shared data +kernel.action_count=2 +kernel.set_up(epsilon=0.01,pool_size=10000,batch=64,update_step=10) +kernel.PO=3 #use PO3 +pool_lock=[Lock(),Lock(),Lock(),Lock(),Lock()] +lock=[Lock(),Lock(),Lock()] +g_lock=Lock() +for p in range(5): + Process(target=kernel.train,args=(p,100,lock,pool_lock)).start() ``` @@ -532,26 +286,6 @@ t.test(dqn,tf,2) ``` -# Online training: -**You can download neural network example in this link,and then you can import neural network and train with kernel,link and example code are below.** - -https://github.com/NoteDancing/Note-documentation/blob/Note-7.0-pv/Note%207.0%20pv%20documentation/DL/neural%20network/tensorflow/nn_ol.py - -**example:** -```python -import Note.DL.kernel as k #import kernel #import platform -import nn_ol as n #import neural network -mnist=tf.keras.datasets.mnist -(x_train,y_train),(x_test,y_test)=mnist.load_data() -x_train,x_test =x_train/255.0,x_test/255.0 -nn=n.nn(x_train,y_train) #create neural network object -kernel=k.kernel(nn) #start kernel -kernel.platform=tf #use platform -kernel.data(x_train,y_train) #input train data -kernel.train_ol() #train neural network -``` - - # Note Compiler: documentation:https://github.com/NoteDancing/Note-documentation/tree/Note-7.0-pv/Note%207.0%20pv%20documentation/compiler ```python From 974089c2e0c70b5654d29f3bbfec2976c8278fc9 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Fri, 30 Jun 2023 21:30:27 +0800 Subject: [PATCH 015/337] Delete nn_attenuate.py --- .../neural network/tensorflow/nn_attenuate.py | 32 ------------------- 1 file changed, 32 deletions(-) delete mode 100644 Note 7.0 pv documentation/DL/neural network/tensorflow/nn_attenuate.py diff --git a/Note 7.0 pv documentation/DL/neural network/tensorflow/nn_attenuate.py b/Note 7.0 pv documentation/DL/neural network/tensorflow/nn_attenuate.py deleted file mode 100644 index 98cc31e9..00000000 --- a/Note 7.0 pv documentation/DL/neural network/tensorflow/nn_attenuate.py +++ /dev/null @@ -1,32 +0,0 @@ -import tensorflow as tf - - -#gradient attenuation example -class nn: - def __init__(self): - self.model=tf.keras.models.Sequential([ - tf.keras.layers.Flatten(input_shape=(28, 28)), - tf.keras.layers.Dense(128,activation='relu'), - tf.keras.layers.Dropout(0.2), - tf.keras.layers.Dense(10) - ]) - self.param=self.model.weights - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.opt=tf.keras.optimizers.Adam(learning_rate=0.001) - self.opt_counter=tf.Variable(tf.zeros(3,dtype=tf.float64)) - - - def fp(self,data): - output=self.model(data) - return output - - - def loss(self,output,labels): - return self.loss_object(labels,output) - - - def attenuate(self,gradient,oc,t): #gradient attenuation function,kernel uses it to calculate attenuation coefficient. - ac=0.9**oc[t] #ac:attenuation coefficient - for i in range(len(gradient)): #oc:optimizing counter - gradient[i]=ac*gradient[i] #t:thread number - return gradient From 83cb99b03ea8f473afabf577d4a670c7ecedbce0 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Fri, 30 Jun 2023 21:30:54 +0800 Subject: [PATCH 016/337] Delete nn_device.py --- .../DL/neural network/tensorflow/nn_device.py | 27 ------------------- 1 file changed, 27 deletions(-) delete mode 100644 Note 7.0 pv documentation/DL/neural network/tensorflow/nn_device.py diff --git a/Note 7.0 pv documentation/DL/neural network/tensorflow/nn_device.py b/Note 7.0 pv documentation/DL/neural network/tensorflow/nn_device.py deleted file mode 100644 index 6b7b1135..00000000 --- a/Note 7.0 pv documentation/DL/neural network/tensorflow/nn_device.py +++ /dev/null @@ -1,27 +0,0 @@ -import tensorflow as tf - - -class nn: #A neural network class example,allocate device for multiple threads. - def __init__(self): - self.weight1=tf.Variable(tf.random.normal([784,64])) - self.bias1=tf.Variable(tf.random.normal([64])) - self.weight2=tf.Variable(tf.random.normal([64,64])) - self.bias2=tf.Variable(tf.random.normal([64])) - self.weight3=tf.Variable(tf.random.normal([64,10])) - self.bias3=tf.Variable(tf.random.normal([10])) - self.param=[self.weight1,self.weight2,self.weight3,self.bias1,self.bias2,self.bias3] - self.optimizer=tf.keras.optimizers.Adam() - self.device_table={0:'GPU:0',1:'GPU:0',2:'GPU:0',3:'GPU:1',4:'GPU:1',5:'GPU:1',6:'GPU:2'} - self.info='example' - - - def fp(self,data,t): - with tf.device(self.device_table[t]): - layer1=tf.nn.relu(tf.matmul(data,self.param[0])+self.param[3]) - layer2=tf.nn.relu(tf.matmul(layer1,self.param[1])+self.param[4]) - output=tf.matmul(layer2,self.param[2])+self.param[5] - return output - - - def loss(self,output,labels): - return tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=output,labels=labels)) From 0055ae029cc12bdc15e14004c12f89d0a1696335 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Fri, 30 Jun 2023 21:31:11 +0800 Subject: [PATCH 017/337] Delete nn_ol.py --- .../DL/neural network/tensorflow/nn_ol.py | 37 ------------------- 1 file changed, 37 deletions(-) delete mode 100644 Note 7.0 pv documentation/DL/neural network/tensorflow/nn_ol.py diff --git a/Note 7.0 pv documentation/DL/neural network/tensorflow/nn_ol.py b/Note 7.0 pv documentation/DL/neural network/tensorflow/nn_ol.py deleted file mode 100644 index 5a0beb8a..00000000 --- a/Note 7.0 pv documentation/DL/neural network/tensorflow/nn_ol.py +++ /dev/null @@ -1,37 +0,0 @@ -import tensorflow as tf -import numpy as np - -#This is an example of online training. -class nn: - def __init__(self,data,labels): - self.train_data=data - self.train_labels=labels - self.model=tf.keras.models.Sequential([ - tf.keras.layers.Flatten(input_shape=(28, 28)), - tf.keras.layers.Dense(128,activation='relu'), - tf.keras.layers.Dropout(0.2), - tf.keras.layers.Dense(10) - ]) - self.param=self.model.weights #parameter list,kernel uses it list for backpropagation. - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.opt=tf.keras.optimizers.Adam() #optimizer,kernel uses it to optimize. - self.train_loss_list=[] - self.c=0 #counter - self.max_length=1000 - - - def ol(self): #This is simulative online function. - index=np.random.choice(60000,size=[32]) - if self.c==10000: - return 'stop' - else: - return [self.train_data[index],self.train_labels[index]] - - - def fp(self,data): #forward propagation function,kernel uses it for forward propagation. - output=self.model(data) - return output - - - def loss(self,output,labels): #loss functino,kernel uses it to calculate loss. - return self.loss_object(labels,output) From fcbc3e655395ff8c47a5dece45d954aa45226d46 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Fri, 30 Jun 2023 21:31:19 +0800 Subject: [PATCH 018/337] Delete nn_ol_p.py --- .../DL/neural network/tensorflow/nn_ol_p.py | 37 ------------------- 1 file changed, 37 deletions(-) delete mode 100644 Note 7.0 pv documentation/DL/neural network/tensorflow/nn_ol_p.py diff --git a/Note 7.0 pv documentation/DL/neural network/tensorflow/nn_ol_p.py b/Note 7.0 pv documentation/DL/neural network/tensorflow/nn_ol_p.py deleted file mode 100644 index e7d0eb90..00000000 --- a/Note 7.0 pv documentation/DL/neural network/tensorflow/nn_ol_p.py +++ /dev/null @@ -1,37 +0,0 @@ -import tensorflow as tf -import numpy as np - -#This is an example of online training. -class nn: - def __init__(self,data,labels): - self.train_data=data - self.train_labels=labels - self.model=tf.keras.models.Sequential([ - tf.keras.layers.Flatten(input_shape=(28, 28)), - tf.keras.layers.Dense(128,activation='relu'), - tf.keras.layers.Dropout(0.2), - tf.keras.layers.Dense(10) - ]) - self.param=self.model.weights #parameter list,kernel uses it list for backpropagation. - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.opt=tf.keras.optimizers.Adam() #optimizer,kernel uses it to optimize. - self.train_loss_list=[] - self.c=0 #counter - self.max_length=1000 - - - def ol(self,t): #This is simulative online function. - index=np.random.choice(60000,size=[32]) - if self.c>=10000: - return 'stop' - else: - return [self.train_data[index],self.train_labels[index]] - - - def fp(self,data): #forward propagation function,kernel uses it for forward propagation. - output=self.model(data) - return output - - - def loss(self,output,labels): #loss functino,kernel uses it to calculate loss. - return self.loss_object(labels,output) From a068b4f2e9781268d302b4c06fbac859b4334072 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Fri, 30 Jun 2023 21:35:53 +0800 Subject: [PATCH 019/337] Add files via upload --- .../DL/neural network/tensorflow/nn_ol.py | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 Note 7.0 pv documentation/DL/neural network/tensorflow/nn_ol.py diff --git a/Note 7.0 pv documentation/DL/neural network/tensorflow/nn_ol.py b/Note 7.0 pv documentation/DL/neural network/tensorflow/nn_ol.py new file mode 100644 index 00000000..c32bc542 --- /dev/null +++ b/Note 7.0 pv documentation/DL/neural network/tensorflow/nn_ol.py @@ -0,0 +1,37 @@ +import tensorflow as tf +import numpy as np + +#This is an example of online training. +class nn: + def __init__(self,data,labels): + self.train_data=data + self.train_labels=labels + self.model=tf.keras.models.Sequential([ + tf.keras.layers.Flatten(input_shape=(28, 28)), + tf.keras.layers.Dense(128,activation='relu'), + tf.keras.layers.Dropout(0.2), + tf.keras.layers.Dense(10) + ]) + self.param=self.model.weights #parameter list,kernel uses it list for backpropagation. + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.opt=tf.keras.optimizers.Adam() #optimizer,kernel uses it to optimize. + self.train_loss_list=[] + self.c=0 #counter + self.max_length=1000 + + + def ol(self): #This is simulative online function. + index=np.random.choice(60000,size=[32]) + if self.c==10000: + return 'stop' + else: + return [self.train_data[index],self.train_labels[index]] + + + def fp(self,data): #forward propagation function,kernel uses it for forward propagation. + output=self.model(data) + return output + + + def loss(self,output,labels): #loss functino,kernel uses it to calculate loss. + return self.loss_object(labels,output) \ No newline at end of file From 84b4ddb493129dafa389510b3a35ff2c85f83589 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Fri, 30 Jun 2023 21:37:29 +0800 Subject: [PATCH 020/337] Update README.md --- README.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/README.md b/README.md index 90fb5e36..07e6a056 100644 --- a/README.md +++ b/README.md @@ -261,6 +261,26 @@ for p in range(5): ``` +# Online training: +**You can download neural network example in this link,and then you can import neural network and train with kernel,link and example code are below.** + +https://github.com/NoteDancing/Note-documentation/blob/Note-7.0-pv/Note%207.0%20pv%20documentation/DL/neural%20network/tensorflow/nn_ol.py + +**example:** +```python +import Note.DL.kernel as k #import kernel #import platform +import nn_ol as n #import neural network +mnist=tf.keras.datasets.mnist +(x_train,y_train),(x_test,y_test)=mnist.load_data() +x_train,x_test =x_train/255.0,x_test/255.0 +nn=n.nn(x_train,y_train) #create neural network object +kernel=k.kernel(nn) #start kernel +kernel.platform=tf #use platform +kernel.data(x_train,y_train) #input train data +kernel.train_ol() #train neural network +``` + + # Test neural network: ## DL: You can test it before using the kernel training neural network. From bdf3717b5b6b127291e6d738367e8eef62f21b78 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Fri, 30 Jun 2023 23:38:27 +0800 Subject: [PATCH 021/337] Update README.md --- README.md | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 07e6a056..32f05f19 100644 --- a/README.md +++ b/README.md @@ -38,27 +38,22 @@ kernel.visualize_reward() # Parallel test: **You can download neural network example in this link,and then you can import neural network and train with kernel,link and example code are below.** -https://github.com/NoteDancing/Note-documentation/blob/Note-7.0-pv/Note%207.0%20pv%20documentation/DL/neural%20network/tensorflow/nn_acc.py +https://github.com/NoteDancing/Note-documentation/blob/Note-7.0-pv/Note%207.0%20pv%20documentation/DL/neural%20network/tensorflow/process/nn.py ```python +import tensorflow as tf import nn_acc as n import Note.DL.dl.test as t -import threading mnist=tf.keras.datasets.mnist (x_train,y_train),(x_test,y_test)=mnist.load_data() x_train,x_test =x_train/255.0,x_test/255.0 nn=n.nn() +nn.build() test=t.parallel_test(nn,x_test,y_test,6,32) test.segment_data() -class thread(threading.Thread): - def run(self): - test.test() -for _ in range(6): - _thread=thread() - _thread.start() -for _ in range(6): - _thread.join() -loss,acc=test.loss_acc() +for p in range(6): + Process(target=test.test).start() +loss=test.loss_acc() ``` From 7388c2fb3f67f6e13a4aa45e9ef342841d163f8c Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Sat, 1 Jul 2023 14:23:53 +0800 Subject: [PATCH 022/337] Update nn_acc.py --- .../DL/neural network/tensorflow/process/nn_acc.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Note 7.0 pv documentation/DL/neural network/tensorflow/process/nn_acc.py b/Note 7.0 pv documentation/DL/neural network/tensorflow/process/nn_acc.py index 65311a69..0647360e 100644 --- a/Note 7.0 pv documentation/DL/neural network/tensorflow/process/nn_acc.py +++ b/Note 7.0 pv documentation/DL/neural network/tensorflow/process/nn_acc.py @@ -2,13 +2,13 @@ import Note.nn.layer.dense as d from Note.nn.process.optimizer import Momentum from Note.nn.layer.flatten import flatten +from Note.nn.accuracy import sparse_categorical_accuracy # Define a neural network class class nn: def __init__(self): # Initialize the loss function and the optimizer self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.train_accuracy=tf.keras.metrics.SparseCategoricalAccuracy(name='train_accuracy') self.optimizer=Momentum(0.07,0.7) @@ -35,10 +35,10 @@ def loss(self,output,labels): def accuracy(self,output,labels): #accuracy function,kernel uses it to calculate accuracy. - return self.train_accuracy(labels,output) + return sparse_categorical_accuracy(labels,output) def opt(self,gradient): # Perform optimization on the parameters using the gradient param=self.optimizer.opt(gradient,self.param) - return param \ No newline at end of file + return param From 67f38586bfa4658bb58e2724e6c2dc464065da28 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Sat, 1 Jul 2023 19:58:26 +0800 Subject: [PATCH 023/337] Add files via upload --- reduced kernel/DL/kernel_reduced.py | 741 ++++++++++++++++++++++++++++ 1 file changed, 741 insertions(+) create mode 100644 reduced kernel/DL/kernel_reduced.py diff --git a/reduced kernel/DL/kernel_reduced.py b/reduced kernel/DL/kernel_reduced.py new file mode 100644 index 00000000..392d5897 --- /dev/null +++ b/reduced kernel/DL/kernel_reduced.py @@ -0,0 +1,741 @@ +from tensorflow import function +import numpy as np +import matplotlib.pyplot as plt +import time + + +class kernel: + def __init__(self,nn=None): + self.nn=nn # the neural network object + try: + self.nn.km=1 # a flag to indicate the kernel mode + except Exception: + pass + self.platform=None # the platform to use, either tensorflow or pytorch + self.batches=None # the number of batches for training data + self.suspend=False # a flag to indicate whether to suspend the training + self.stop=False # a flag to indicate whether to stop the training + self.stop_flag=False # a flag to indicate whether the training has been stopped + self.save_epoch=None # the epoch number to save the model + self.batch=None # the batch size for training data + self.epoch=0 # the current epoch number + self.end_loss=None # the target loss value to end the training + self.end_acc=None # the target accuracy value to end the training + self.end_test_loss=None # the target test loss value to end the training + self.end_test_acc=None # the target test accuracy value to end the training + self.acc_flag='%' # a flag to indicate whether to use percentage or decimal for accuracy display + self.train_counter=0 # a counter for how many times the train method has been called + self.filename='save.dat' # the file name to save the model + self.train_loss=None # the current train loss value + self.train_acc=None # the current train accuracy value + self.train_loss_list=[] # a list of train loss values for each epoch + self.train_acc_list=[] # a list of train accuracy values for each epoch + self.test_loss=None # the current test loss value + self.test_acc=None # the current test accuracy value + self.test_loss_list=[] # a list of test loss values for each epoch + self.test_acc_list=[] # a list of test accuracy values for each epoch + self.test_flag=False # a flag to indicate whether to use test data or not + self.total_epoch=0 # the total number of epochs for all trainings + self.time=0 # the time elapsed for one training session + self.total_time=0 # the total time elapsed for all trainings + + + def data(self,train_data=None,train_labels=None,test_data=None,test_labels=None,train_dataset=None,test_dataset=None): + if type(self.nn.param[0])!=list: + self.train_data=train_data.astype(self.nn.param[0].dtype.name) # convert the train data type to match the neural network parameters type + self.train_labels=train_labels.astype(self.nn.param[0].dtype.name) # convert the train labels type to match the neural network parameters type + else: + self.train_data=train_data.astype(self.nn.param[0][0].dtype.name) # convert the train data type to match the neural network parameters type (for multiple inputs) + self.train_labels=train_labels.astype(self.nn.param[0][0].dtype.name) # convert the train labels type to match the neural network parameters type (for multiple inputs) + self.train_dataset=train_dataset # a tensorflow or pytorch dataset object for train data (optional) + if test_data is not None: + if type(self.nn.param[0])!=list: + self.test_data=test_data.astype(self.nn.param[0].dtype.name) # convert the test data type to match the neural network parameters type + self.test_labels=test_labels.astype(self.nn.param[0].dtype.name) # convert the test labels type to match the neural network parameters type + else: + self.test_data=test_data.astype(self.nn.param[0][0].dtype.name) # convert the test data type to match the neural network parameters type (for multiple inputs) + self.test_labels=test_labels.astype(self.nn.param[0][0].dtype.name) # convert the test labels type to match the neural network parameters type (for multiple inputs) + self.test_flag=True # set the test flag to True if test data is provided + self.test_dataset=test_dataset # a tensorflow or pytorch dataset object for test data (optional) + if self.train_dataset==None: + self.shape0=train_data.shape[0] # get the number of samples in train data + return + + + def init(self): # a method to initialize the attributes for a new training session + self.suspend=False + self.stop=False + self.stop_flag=False + self.save_epoch=None + self.end_loss=None + self.end_acc=None + self.end_test_loss=None + self.end_test_acc=None + self.train_loss=None + self.train_acc=None + self.test_loss=None + self.test_acc=None + self.train_loss_list.clear() + self.train_acc_list.clear() + self.test_loss_list.clear() + self.test_acc_list.clear() + self.test_flag=False + self.train_counter=0 + self.epoch=0 + self.total_epoch=0 + self.time=0 + self.total_time=0 + return + + + def end(self): # a method to check whether the training has reached the target loss or accuracy values + if self.end_loss!=None and len(self.train_loss_list)!=0 and self.train_loss_list[-1]self.end_acc: # if the target train accuracy is given and the current train accuracy is higher than it + return True + elif self.end_loss!=None and len(self.train_loss_list)!=0 and self.end_acc!=None and self.train_loss_list[-1]self.end_acc: # if both the target train loss and accuracy are given and the current train loss is lower than it and the current train accuracy is higher than it + return True + elif self.end_test_loss!=None and len(self.test_loss_list)!=0 and self.test_loss_list[-1]self.end_test_acc: # if the target test accuracy is given and the current test accuracy is higher than it + return True + elif self.end_test_loss!=None and len(self.test_loss_list)!=0 and self.end_test_acc!=None and self.test_loss_list[-1]self.end_test_acc: # if both the target test loss and accuracy are given and the current test loss is lower than it and the current test accuracy is higher than it + return True + + + def loss_acc(self,output=None,labels_batch=None,loss=None,test_batch=None,total_loss=None,total_acc=None): # a method to calculate the loss and accuracy values for each batch or epoch + if self.batch!=None: # if batch mode is used + total_loss+=loss # accumulate the batch loss to total loss + try: + batch_acc=self.nn.accuracy(output,labels_batch) # calculate the batch accuracy using the neural network's accuracy method + total_acc+=batch_acc # accumulate the batch accuracy to total accuracy + except Exception as e: + try: + if self.nn.accuracy!=None: # if the neural network has an accuracy method defined + raise e # raise the exception + except Exception: + pass + return total_loss,total_acc # return the total loss and accuracy values for all batches so far + else: # if batch mode is not used (use all data at once) + loss=loss.numpy() # convert the loss value to numpy array + self.train_loss=loss # assign the loss value to train loss attribute + self.train_loss_list.append(loss) # append the loss value to train loss list + try: + acc=self.nn.accuracy(output,self.train_labels) # calculate the accuracy value using the neural network's accuracy method + acc=acc.numpy() # convert the accuracy value to numpy array + self.train_acc=acc # assign the accuracy value to train accuracy attribute + self.train_acc_list.append(acc) # append the accuracy value to train accuracy list + except Exception as e: + try: + if self.nn.accuracy!=None: # if the neural network has an accuracy method defined + raise e # raise the exception + except Exception: + pass + if self.test_flag==True: # if test data is used + self.test_loss,self.test_acc=self.test(self.test_data,self.test_labels,test_batch) # calculate the test loss and accuracy values using the test method + self.test_loss_list.append(self.test_loss) # append the test loss value to test loss list + try: + self.test_acc_list.append(self.test_acc) # append the test accuracy value to test accuracy list + except Exception as e: + try: + if self.nn.accuracy!=None: # if the neural network has an accuracy method defined + raise e # raise the exception + except Exception: + pass + return # return nothing for this case + + + def data_func(self,data_batch=None,labels_batch=None,batch=None,index1=None,index2=None,j=None,flag=None): # a method to get a batch of data and labels from the train data and labels + if flag==None: # if flag is None, it means the batch size is smaller than the number of samples + if batch!=1: # if batch size is not 1 + data_batch=self.train_data[index1:index2] # get a slice of train data according to the index range + else: # if batch size is 1 + data_batch=self.train_data[j] # get one sample of train data according to the index + if batch!=1: # if batch size is not 1 + labels_batch=self.train_labels[index1:index2] # get a slice of train labels according to the index range + else: # if batch size is 1 + labels_batch=self.train_labels[j] # get one sample of train labels according to the index + else: # if flag is not None, it means the batch size is larger than the number of samples + try: + try: + data_batch=self.platform.concat([self.train_data[index1:],self.train_data[:index2]],0) # concatenate two slices of train data from the end and the beginning to form a batch + labels_batch=self.platform.concat([self.train_labels[index1:],self.train_labels[:index2]],0) # concatenate two slices of train labels from the end and the beginning to form a batch + except Exception: # if the platform's concat method fails + data_batch=np.concatenate([self.train_data[index1:],self.train_data[:index2]],0) # use numpy's concatenate method instead + labels_batch=np.concatenate([self.train_labels[index1:],self.train_labels[:index2]],0) # use numpy's concatenate method instead + except Exception as e: + raise e # raise any other exception + return data_batch,labels_batch # return the batch of data and labels + + + @function(jit_compile=True) # use tensorflow's function decorator to speed up the execution + def tf_opt(self,data,labels): # a method to perform one optimization step using tensorflow platform + try: + try: + if self.nn.GradientTape!=None: # if the neural network has a GradientTape method defined + tape,output,loss=self.nn.GradientTape(data,labels) # use the neural network's GradientTape method to get the tape, output and loss values + except Exception: # if the neural network does not have a GradientTape method defined or it fails + with self.platform.GradientTape(persistent=True) as tape: # use tensorflow's GradientTape context manager instead + try: + output=self.nn.fp(data) # get the output value using the neural network's forward propagation method + loss=self.nn.loss(output,labels) # get the loss value using the neural network's loss function + except Exception: # if the neural network's forward propagation method or loss function fails or they are combined in one method + output,loss=self.nn.fp(data,labels) # use the neural network's forward propagation method with both data and labels as inputs to get the output and loss values + except Exception as e: + raise e # raise any other exception + try: + try: + gradient=self.nn.gradient(tape,loss) # use the neural network's gradient method to get the gradient value from the tape and loss values + except Exception: # if the neural network does not have a gradient method defined or it fails + gradient=tape.gradient(loss,self.nn.param) # use tensorflow's tape.gradient method instead with loss value and neural network parameters as inputs + except Exception as e: + raise e # raise any other exception + try: + try: + self.nn.opt.apply_gradients(zip(gradient,self.nn.param)) # use the neural network's optimizer's apply_gradients method to update the neural network parameters with gradient value + except Exception: # if the neural network does not have an optimizer or its apply_gradients method fails + self.nn.opt(gradient) # use the neural network's optimizer directly with gradient value as input + except Exception as e: + raise e # raise any other exception + return output,loss # return the output and loss values for this optimization step + + + def pytorch_opt(self,data,labels): # a method to perform one optimization step using pytorch platform + output=self.nn.fp(data) # get the output value using the neural network's forward propagation method + loss=self.nn.loss(output,labels) # get the loss value using the neural network's loss function + try: + try: + self.nn.opt.zero_grad() # use the neural network's optimizer's zero_grad method to clear the previous gradients + loss.backward() # use pytorch's loss.backward method to calculate the gradients + self.nn.opt.step() # use the neural network's optimizer's step method to update the neural network parameters with gradient value + except Exception: # if the neural network does not have an optimizer or its zero_grad or step method fails + self.nn.opt(loss) # use the neural network's optimizer directly with loss value as input + except Exception as e: + raise e # raise any other exception + return output,loss # return the output and loss values for this optimization step + + + def opt(self,data,labels): # a method to perform one optimization step using either tensorflow or pytorch platform + try: + try: + if self.platform.DType!=None: # if tensorflow platform is used + output,loss=self.tf_opt(data,labels) # use the tf_opt method for optimization + except Exception: # if tensorflow platform is not used or it fails + output,loss=self.pytorch_opt(data,labels) # use the pytorch_opt method for optimization + except Exception as e: + raise e # raise any other exception + return output,loss # return the output and loss values for this optimization step + + + def _train(self,batch=None,test_batch=None): # a method to perform one epoch of training + if batch!=None: # if batch mode is used + total_loss=0 # initialize the total loss value for all batches + total_acc=0 # initialize the total accuracy value for all batches + if self.train_dataset!=None: # if a tensorflow or pytorch dataset object is used for train data + for data_batch,labels_batch in self.train_dataset: # iterate over each batch of data and labels from the train dataset + if self.stop==True: # if the stop flag is set to True + if self.stop_func(): # check whether the training has reached the target loss or accuracy values using the stop_func method + return # return nothing and end the training + try: + data_batch,labels_batch=self.nn.data_func(data_batch,labels_batch) # use the neural network's data_func method to preprocess the data and labels batch (optional) + except Exception as e: + try: + if self.nn.data_func!=None: # if the neural network has a data_func method defined + raise e # raise the exception + except Exception: + pass + output,batch_loss=self.opt(data_batch,labels_batch) # perform one optimization step using the opt method and get the output and batch loss values + total_loss,total_acc=self.loss_acc(output=output,labels_batch=labels_batch,loss=batch_loss,total_loss=total_loss,total_acc=total_acc) # calculate and update the total loss and accuracy values for all batches using the loss_acc method + else: # if a numpy array is used for train data + total_loss=0 # initialize the total loss value for all batches + total_acc=0 # initialize the total accuracy value for all batches + batches=int((self.shape0-self.shape0%batch)/batch) # calculate how many batches are needed for train data according to batch size + for j in range(batches): # iterate over each batch index + if self.stop==True: # if the stop flag is set to True + if self.stop_func(): # check whether the training has reached the target loss or accuracy values using the stop_func method + return # return nothing and end the training + index1=j*batch # calculate the start index of train data for this batch + index2=(j+1)*batch # calculate the end index of train data for this batch + data_batch,labels_batch=self.data_func(batch,index1,index2,j) # get a batch of data and labels from train data and labels using the data_func method + try: + data_batch,labels_batch=self.nn.data_func(data_batch,labels_batch) # use the neural network's data_func method to preprocess the data and labels batch (optional) + except Exception as e: + try: + if self.nn.data_func!=None: # if the neural network has a data_func method defined + raise e # raise the exception + except Exception: + pass + output,batch_loss=self.opt(data_batch,labels_batch) # perform one optimization step + total_loss,total_acc=self.loss_acc(output=output,labels_batch=labels_batch,loss=batch_loss,total_loss=total_loss,total_acc=total_acc) # calculate and update the total loss and accuracy values for all batches using the loss_acc method + try: + try: + self.nn.bc.assign_add(1) # use the neural network's batch counter's assign_add method to increase the batch counter by 1 (for tensorflow platform) + except Exception: # if the neural network does not have a batch counter or its assign_add method fails + self.nn.bc+=1 # use the normal addition operation to increase the batch counter by 1 (for pytorch platform) + except Exception: + pass + if self.shape0%batch!=0: # if there are some samples left in train data that are not enough to form a full batch + if self.stop==True: # if the stop flag is set to True + if self.stop_func(): # check whether the training has reached the target loss or accuracy values using the stop_func method + return # return nothing and end the training + batches+=1 # increase the number of batches by 1 to include the remaining samples + index1=batches*batch # calculate the start index of train data for the last batch + index2=batch-(self.shape0-batches*batch) # calculate how many samples are needed from the beginning of train data to form a full batch + data_batch,labels_batch=self.data_func(batch,index1,index2,flag=True) # get a batch of data and labels from train data and labels using the data_func method with flag set to True + try: + data_batch,labels_batch=self.nn.data_func(data_batch,labels_batch) # use the neural network's data_func method to preprocess the data and labels batch (optional) + except Exception as e: + try: + if self.nn.data_func!=None: # if the neural network has a data_func method defined + raise e # raise the exception + except Exception: + pass + output,batch_loss=self.opt(data_batch,labels_batch) # perform one optimization step using the opt method and get the output and batch loss values + total_loss,total_acc=self.loss_acc(output=output,labels_batch=labels_batch,loss=batch_loss,total_loss=total_loss,total_acc=total_acc) # calculate and update the total loss and accuracy values for all batches using the loss_acc method + try: + try: + self.nn.bc.assign_add(1) # use the neural network's batch counter's assign_add method to increase the batch counter by 1 (for tensorflow platform) + except Exception: # if the neural network does not have a batch counter or its assign_add method fails + self.nn.bc+=1 # use the normal addition operation to increase the batch counter by 1 (for pytorch platform) + except Exception: + pass + try: + if self.platform.DType!=None: # if tensorflow platform is used + loss=total_loss.numpy()/batches # convert the total loss value to numpy array and divide it by number of batches to get the average loss value for this epoch + except Exception: # if tensorflow platform is not used or it fails + loss=total_loss.detach().numpy()/batches # detach the total loss value from computation graph and convert it to numpy array and divide it by number of batches to get the average loss value for this epoch + try: + train_acc=total_acc.numpy()/batches # convert the total accuracy value to numpy array and divide it by number of batches to get the average accuracy value for this epoch + except Exception as e: + try: + if self.nn.accuracy!=None: # if the neural network has an accuracy method defined + raise e # raise the exception + except Exception: + pass + self.train_loss=loss # assign the average loss value to train loss attribute + self.train_loss_list.append(loss) # append the average loss value to train loss list + try: + self.train_acc=train_acc # assign the average accuracy value to train accuracy attribute + self.train_acc_list.append(train_acc) # append the average accuracy value to train accuracy list + except Exception as e: + try: + if self.nn.accuracy!=None: # if the neural network has an accuracy method defined + raise e # raise the exception + except Exception: + pass + if self.test_flag==True: # if test data is used + self.test_loss,self.test_acc=self.test(self.test_data,self.test_labels,test_batch) # calculate the test loss and accuracy values using the test method + self.test_loss_list.append(self.test_loss) # append the test loss value to test loss list + try: + self.test_acc_list.append(self.test_acc) # append the test accuracy value to test accuracy list + except Exception as e: + try: + if self.nn.accuracy!=None: # if the neural network has an accuracy method defined + raise e # raise the exception + except Exception: + pass + else: # if batch mode is not used (use all data at once) + output,train_loss=self.opt(self.train_data,self.train_labels) # perform one optimization step using the opt method and get the output and train loss values + self.loss_acc(output=output,labels_batch=labels_batch,loss=train_loss,test_batch=test_batch,total_loss=total_loss,total_acc=total_acc) # calculate and update the train and test loss and accuracy values using the loss_acc method + return # return nothing for this case + + + def train(self,batch=None,epoch=None,test_batch=None,save=None,one=True,p=None,s=None): # a method to perform multiple epochs of training + self.batch=batch # assign the batch size for training data to batch attribute + self.epoch=0 # initialize the current epoch number to 0 + self.train_counter+=1 # increase the train counter by 1 + if p==None: # if p is None, it means the default value of p is used + self.p=9 # assign 9 to p attribute, which means print the train and test loss and accuracy values every 10 epochs + else: + self.p=p-1 # assign p-1 to p attribute, which means print the train and test loss and accuracy values every p epochs + if s==None: # if s is None, it means the default value of s is used + self.s=1 # assign 1 to s attribute, which means save the model every epoch + self.file_list=None # assign None to file_list attribute, which means do not keep track of saved files + else: + self.s=s-1 # assign s-1 to s attribute, which means save the model every s epochs + self.file_list=[] # assign an empty list to file_list attribute, which means keep track of saved files + if epoch!=None: # if epoch is not None, it means a fixed number of epochs is given + for i in range(epoch): # iterate over each epoch index + t1=time.time() # record the start time of this epoch + self._train(batch,test_batch) # perform one epoch of training using the _train method + if self.stop_flag==True: # if the stop flag is set to True, it means the training has reached the target loss or accuracy values and has been stopped + return # return nothing and end the training + try: + try: + self.nn.ec.assign_add(1) # use the neural network's epoch counter's assign_add method to increase the epoch counter by 1 (for tensorflow platform) + except Exception: # if the neural network does not have an epoch counter or its assign_add method fails + self.nn.ec+=1 # use the normal addition operation to increase the epoch counter by 1 (for pytorch platform) + except Exception: + pass + self.total_epoch+=1 # increase the total epoch number by 1 for all trainings + if epoch%10!=0: + p=epoch-epoch%self.p + p=int(p/self.p) + s=epoch-epoch%self.s + s=int(s/self.s) + else: + p=epoch/(self.p+1) + p=int(p) + s=epoch/(self.s+1) + s=int(s) + if p==0: + p=1 + if s==0: + s=1 + if i%p==0: # if this epoch index is a multiple of p (or p+1) + if self.test_flag==False: # if test data is not used + try: + if self.nn.accuracy!=None: # if the neural network has an accuracy method defined + print('epoch:{0} loss:{1:.6f}'.format(i+1,self.train_loss)) # print the current epoch number and train loss value + if self.acc_flag=='%': # if percentage mode is used for accuracy display + print('epoch:{0} accuracy:{1:.1f}'.format(i+1,self.train_acc*100)) # print the current epoch number and train accuracy value in percentage + else: # if decimal mode is used for accuracy display + print('epoch:{0} accuracy:{1:.6f}'.format(i+1,self.train_acc)) # print the current epoch number and train accuracy value in print('epoch:{0} accuracy:{1:.6f}'.format(i+1,self.train_acc)) # print the current epoch number and train accuracy value in decimal + print() # print an empty line for readability + except Exception: # if the neural network does not have an accuracy method defined or it fails + print('epoch:{0} loss:{1:.6f}'.format(i+1,self.train_loss)) # print the current epoch number and train loss value + print() # print an empty line for readability + else: # if test data is used + try: + if self.nn.accuracy!=None: # if the neural network has an accuracy method defined + print('epoch:{0} loss:{1:.6f},test loss:{2:.6f}'.format(i+1,self.train_loss,self.test_loss)) # print the current epoch number, train loss value and test loss value + if self.acc_flag=='%': # if percentage mode is used for accuracy display + print('epoch:{0} accuracy:{1:.1f},test accuracy:{2:.1f}'.format(i+1,self.train_acc*100,self.test_acc*100)) # print the current epoch number, train accuracy value and test accuracy value in percentage + else: # if decimal mode is used for accuracy display + print('epoch:{0} accuracy:{1:.6f},test accuracy:{2:.6f}'.format(i+1,self.train_acc,self.test_acc)) # print the current epoch number, train accuracy value and test accuracy value in decimal + print() # print an empty line for readability + except Exception: # if the neural network does not have an accuracy method defined or it fails + print('epoch:{0} loss:{1:.6f},test loss:{2:.6f}'.format(i+1,self.train_loss,self.test_loss)) # print the current epoch number, train loss value and test loss value + print() # print an empty line for readability + if save!=None and i%s==0: # if save is not None, it means a file name is given to save the model, and if this epoch index is a multiple of s (or s+1) + self.save(self.total_epoch,one) # save the model using the save method with total epoch number and one flag as inputs + t2=time.time() # record the end time of this epoch + self.time+=(t2-t1) # calculate and update the time elapsed for this training session + else: # if epoch is None, it means an infinite number of epochs is given + i=0 # initialize the epoch index to 0 + while True: # loop indefinitely until stopped by other conditions + t1=time.time() # record the start time of this epoch + self._train(test_batch=test_batch) # perform one epoch of training using the _train method + if self.stop_flag==True: # if the stop flag is set to True, it means the training has reached the target loss or accuracy values and has been stopped + return # return nothing and end the training + i+=1 # increase the epoch index by 1 + try: + try: + self.nn.ec.assign_add(1) # use the neural network's epoch counter's assign_add method to increase the epoch counter by 1 (for tensorflow platform) + except Exception: # if the neural network does not have an epoch counter or its assign_add method fails + self.nn.ec+=1 # use the normal addition operation to increase the epoch counter by 1 (for pytorch platform) + except Exception: + pass + self.total_epoch+=1 # increase the total epoch number by 1 for all trainings + if epoch%10!=0: + p=epoch-epoch%self.p + p=int(p/self.p) + s=epoch-epoch%self.s + s=int(s/self.s) + else: + p=epoch/(self.p+1) + p=int(p) + s=epoch/(self.s+1) + s=int(s) + if p==0: + p=1 + if s==0: + s=1 + if i%p==0: # if this epoch index is a multiple of p (or p+1) + if self.test_flag==False: # if test data is not used + try: + if self.nn.accuracy!=None: # if the neural network has an accuracy method defined + print('epoch:{0} loss:{1:.6f}'.format(i+1,self.train_loss)) # print the current epoch number and train loss value + if self.acc_flag=='%': # if percentage mode is used for accuracy display + print('epoch:{0} accuracy:{1:.1f}'.format(i+1,self.train_acc*100)) # print the current epoch number and train accuracy value in percentage + else: # if decimal mode is used for accuracy display + print('epoch:{0} accuracy:{1:.6f}'.format(i+1,self.train_acc)) # print the current epoch number and train accuracy value in decimal + print() # print an empty line for readability + except Exception: # if the neural network does not have an accuracy method defined or it fails + print('epoch:{0} loss:{1:.6f}'.format(i+1,self.train_loss)) # print the current epoch number and train loss value + print() # print an empty line for readability + else: # if test data is used + try: + if self.nn.accuracy!=None: # if the neural network has an accuracy method defined + print('epoch:{0} loss:{1:.6f},test loss:{2:.6f}'.format(i+1,self.train_loss,self.test_loss)) # print the current epoch number, train loss value and test loss value + if self.acc_flag=='%': # if percentage mode is used for accuracy display + print('epoch:{0} accuracy:{1:.1f},test accuracy:{2:.1f}'.format(i+1,self.train_acc*100,self.test_acc*100)) # print the current epoch number, train accuracy value and test accuracy value in percentage + else: # if decimal mode is used for accuracy display + print('epoch:{0} accuracy:{1:.6f},test accuracy:{2:.6f}'.format(i+1,self.train_acc,self.test_acc)) # print the current epoch number, train accuracy value and test accuracy value in decimal + print() # print an empty line for readability + except Exception: # if the neural network does not have an accuracy method defined or it fails + print('epoch:{0} loss:{1:.6f},test loss:{2:.6f}'.format(i+1,self.train_loss,self.test_loss)) # print the current epoch number, train loss value and test loss value + print() # print an empty line for readability + if save!=None and i%s==0: # if save is not None, it means a file name is given to save the model, and if this epoch index is a multiple of s (or s+1) + self.save(self.total_epoch,one) # save the model using the save method with total epoch number and one flag as inputs + t2=time.time() # record the end time of this epoch + self.time+=(t2-t1) # calculate and update the time elapsed for this training session + if save!=None: # if save is not None, it means a file name is given to save the model + self.save() # save the model using the save method without any inputs (use default values) + self._time=self.time-int(self.time) # get the fractional part of the time elapsed for this training session + if self._time<0.5: # if the fractional part is less than 0.5 + self.time=int(self.time) # round down the time elapsed to integer + else: # if the fractional part is greater than or equal to 0.5 + self.time=int(self.time)+1 # round up the time elapsed to integer + self.total_time+=self.time # calculate and update the total time elapsed for all trainings + if self.test_flag==False: # if test data is not used + print('last loss:{0:.6f}'.format(self.train_loss)) # print the last train loss value + else: # if test data is used + print('last loss:{0:.6f},last test loss:{1:.6f}'.format(self.train_loss,self.test_loss)) # print the last train loss value and test loss value + try: + if self.nn.accuracy!=None: # if the neural network has an accuracy method defined + if self.acc_flag=='%': # if percentage mode is used for accuracy display + if self.test_flag==False: # if test data is not used + print('last accuracy:{0:.1f}'.format(self.train_acc*100)) # print the last train accuracy value in percentage + else: # if test else: # if test data is used + print('last accuracy:{0:.1f},last test accuracy:{1:.1f}'.format(self.train_acc*100,self.test_acc*100)) # print the last train accuracy value and test accuracy value in percentage + else: # if decimal mode is used for accuracy display + if self.test_flag==False: # if test data is not used + print('last accuracy:{0:.6f}'.format(self.train_acc)) # print the last train accuracy value in decimal + else: # if test data is used + print('last accuracy:{0:.6f},last test accuracy:{1:.6f}'.format(self.train_acc,self.test_acc)) # print the last train accuracy value and test accuracy value in decimal + except Exception: # if the neural network does not have an accuracy method defined or it fails + pass + print() # print an empty line for readability + print('time:{0}s'.format(self.time)) # print the time elapsed for this training session + self.training_flag=False # set the training flag to False, which means the training is finished + return # return nothing and end the training + + + def train_ol(self): # a method to perform online learning, which means updating the model with one sample at a time + while True: # loop indefinitely until stopped by other conditions + if self.stop_flag==True: # if the stop flag is set to True, it means the online learning has reached the target loss or accuracy values and has been stopped + return # return nothing and end the online learning + if self.save_flag==True: # if the save flag is set to True, it means a request to save the model has been made + self.save() # save the model using the save method without any inputs (use default values) + self.suspend_func() # check whether a request to suspend the online learning has been made using the suspend_func method + data=self.nn.ol() # get one sample of data and label from the neural network's ol method, which should be defined by the user + if data=='stop': # if the data is 'stop', it means a request to stop the online learning has been made + return # return nothing and end the online learning + elif data=='suspend': # if the data is 'suspend', it means a request to suspend the online learning has been made + self.nn.suspend=True # set the neural network's suspend attribute to True, which means the online learning is suspended + while True: # loop indefinitely until resumed by other conditions + if self.nn.suspend==False: # if the neural network's suspend attribute is set to False, it means a request to resume the online learning has been made + break # break the loop and resume the online learning + continue # continue to the next iteration of online learning + output,loss=self.opt(data[0],data[1]) # perform one optimization step using the opt method and get the output and loss values + loss=loss.numpy() # convert the loss value to numpy array + if len(self.nn.train_loss_list)==self.nn.max_length: # if the train loss list has reached its maximum length, which should be defined by the user + del self.nn.train_loss_list[0] # delete the first element of train loss list + self.nn.train_loss_list.append(loss) # append the loss value to train loss list + try: + train_acc=self.nn.accuracy(output,data[1]) # calculate the accuracy value using the neural network's accuracy method + if len(self.nn.train_acc_list)==self.nn.max_length: # if the train accuracy list has reached its maximum length, which should be defined by the user + del self.nn.train_acc_list[0] # delete the first element of train accuracy list + self.train_acc_list.append(train_acc) # append the accuracy value to train accuracy list + except Exception as e: + try: + if self.nn.accuracy!=None: # if the neural network has an accuracy method defined + raise e # raise the exception + except Exception: + pass + try: + self.nn.c+=1 # increase the neural network's counter by 1, which should be defined by the user to keep track of how many samples have been used for online learning + except Exception: + pass + return # return nothing and end the online learning + + + def test(self,test_data=None,test_labels=None,batch=None): # a method to calculate the test loss and accuracy values using test data and labels + if batch!=None: # if batch mode is used + total_loss=0 # initialize the total loss value for all batches + total_acc=0 # initialize the total accuracy value for all batches + if self.test_dataset!=None: # if a tensorflow or pytorch dataset object is used for test data + for data_batch,labels_batch in self.test_dataset: # iterate over each batch of data and labels from the test dataset + output=self.nn.fp(data_batch) # get the output value using the neural network's forward propagation method + batch_loss=self.nn.loss(output,labels_batch) # get the batch loss value using the neural network's loss function + total_loss+=batch_loss # accumulate the batch loss to total loss + try: + batch_acc=self.nn.accuracy(output,labels_batch) # calculate the batch accuracy using the neural network's accuracy method + total_acc+=batch_acc # accumulate the batch accuracy to total accuracy + except Exception as e: + try: + if self.nn.accuracy!=None: # if the neural network has an accuracy method defined + raise e # raise the exception + except Exception: + pass + else: # if a numpy array is used for test data + total_loss=0 # initialize the total loss value for all batches + total_acc=0 # initialize the total accuracy value for all batches + if type(test_data)==list: # if test data is a list of arrays (for multiple inputs) + batches=int((test_data[0].shape[0]-test_data[0].shape[0]%batch)/batch) # calculate how many batches are needed for test data according to batch size + shape0=test_data[0].shape[0] # get the number of samples in test data + else: # if test data is a single array + batches=int((test_data.shape[0]-test_data.shape[0]%batch)/batch) # calculate how many batches are needed for test data according to batch size + shape0=test_data.shape[0] # get the number of samples in test data + for j in range(batches): # iterate over each batch index + index1=j*batch # calculate the start index of test data for this batch + index2=(j+1)*batch # calculate the end index of test data for this batch + if type(test_data)==list: # if test data is a list of arrays (for multiple inputs) + for i in range(len(test_data)): + data_batch[i]=test_data[i][index1:index2] # get a slice of test data according to the index range for each input array + else: # if test data is a single array + data_batch=test_data[index1:index2] # get a slice of test data according to the index range + if type(test_labels)==list: # if test labels is a list of arrays (for multiple outputs) + for i in range(len(test_labels)): + labels_batch[i]=test_labels[i][index1:index2] # get a slice of test labels according to the index range for each output array + else: # if test labels is a single array + labels_batch=test_labels[index1:index2] # get a slice of test labels according to the index range + output=self.nn.fp(data_batch) # get the output value using the neural network's forward propagation method + batch_loss=self.nn.loss(output,labels_batch) # get the batch loss value using the neural network's loss function + total_loss+=batch_loss # accumulate the batch loss to total loss + try: + batch_acc=self.nn.accuracy(output,labels_batch) # calculate the batch accuracy using the neural network's accuracy method + total_acc+=batch_acc # accumulate the batch accuracy to total accuracy + except Exception as e: + try: + if self.nn.accuracy!=None: # if the neural network has an accuracy method defined + raise e # raise the exception + except Exception: + pass + if shape0%batch!=0: # if there are some samples left in test data that are not enough to form a full batch + batches+=1 # increase the number of batches by 1 to include the remaining samples + index1=batches*batch # calculate the start index of test data for the last batch + index2=batch-(shape0-batches*batch) # calculate how many samples are needed from the beginning of test data to form a full batch + try: + try: + if type(test_data)==list: # if test data is a list of arrays (for multiple if type(test_data)==list: # if test data is a list of arrays (for multiple inputs) + for i in range(len(test_data)): + data_batch[i]=self.platform.concat([test_data[i][index1:],test_data[i][:index2]],0) # concatenate two slices of test data from the end and the beginning to form a batch for each input array + else: # if test data is a single array + data_batch=self.platform.concat([test_data[index1:],test_data[:index2]],0) # concatenate two slices of test data from the end and the beginning to form a batch + if type(test_labels)==list: # if test labels is a list of arrays (for multiple outputs) + for i in range(len(test_labels)): + labels_batch[i]=self.platform.concat([test_labels[i][index1:],test_labels[i][:index2]],0) # concatenate two slices of test labels from the end and the beginning to form a batch for each output array + else: # if test labels is a single array + labels_batch=self.platform.concat([test_labels[index1:],test_labels[:index2]],0) # concatenate two slices of test labels from the end and the beginning to form a batch + except Exception: # if the platform's concat method fails + if type(test_data)==list: # if test data is a list of arrays (for multiple inputs) + for i in range(len(test_data)): + data_batch[i]=np.concatenate([test_data[i][index1:],test_data[i][:index2]],0) # use numpy's concatenate method instead for each input array + else: # if test data is a single array + data_batch=np.concatenate([test_data[index1:],test_data[:index2]],0) # use numpy's concatenate method instead + if type(test_labels)==list: # if test labels is a list of arrays (for multiple outputs) + for i in range(len(test_labels)): + labels_batch[i]=np.concatenate([test_labels[i][index1:],test_labels[i][:index2]],0) # use numpy's concatenate method instead for each output array + else: # if test labels is a single array + labels_batch=np.concatenate([test_labels[index1:],test_labels[:index2]],0) # use numpy's concatenate method instead + except Exception as e: + raise e # raise any other exception + output=self.nn.fp(data_batch) # get the output value using the neural network's forward propagation method + batch_loss=self.nn.loss(output,labels_batch) # get the batch loss value using the neural network's loss function + total_loss+=batch_loss # accumulate the batch loss to total loss + try: + batch_acc=self.nn.accuracy(output,labels_batch) # calculate the batch accuracy using the neural network's accuracy method + total_acc+=batch_acc # accumulate the batch accuracy to total accuracy + except Exception as e: + try: + if self.nn.accuracy!=None: # if the neural network has an accuracy method defined + raise e # raise the exception + except Exception: + pass + test_loss=total_loss.numpy()/batches # convert the total loss value to numpy array and divide it by number of batches to get the average loss value for test data + try: + if self.nn.accuracy!=None: # if the neural network has an accuracy method defined + test_acc=total_acc.numpy()/batches # convert the total accuracy value to numpy array and divide it by number of batches to get the average accuracy value for test data + except Exception: + pass + else: # if batch mode is not used (use all data at once) + output=self.nn.fp(test_data) # get the output value using the neural network's forward propagation method + test_loss=self.nn.loss(output,test_labels) # get the loss value using the neural network's loss function + test_loss=test_loss.numpy() # convert the loss value to numpy array + try: + test_acc=self.nn.accuracy(output,test_labels) # calculate the accuracy value using the neural network's accuracy method + test_acc=test_acc.numpy() # convert the accuracy value to numpy array + except Exception as e: + try: + if self.nn.accuracy!=None: # if the neural network has an accuracy method defined + raise e # raise the exception + except Exception: + pass + try: + if self.nn.accuracy!=None: # if the neural network has an accuracy method defined + return test_loss,test_acc # return the test loss and accuracy values + except Exception: + return test_loss,None # return the test loss value and None + + + def suspend_func(self): # a method to check whether a request to suspend the training or online learning has been made + if self.suspend==True: # if the suspend flag is set to True + if self.save_epoch==None: # if the save epoch number is None, it means no request to save the model has been made + print('Training have suspended.') # print a message to indicate the training or online learning has been suspended + else: + self._save() # save the model using the _save method + while True: # loop indefinitely until resumed by other conditions + if self.suspend==False: # if the suspend flag is set to False, it means a request to resume the training or online learning has been made + print('Training have continued.') # print a message to indicate the training or online learning has been resumed + break # break the loop and resume the training or online learning + return # return nothing for this case + + + def stop_func(self): # a method to check whether the training or online learning has reached the target loss or accuracy values and stop it accordingly + if self.end(): # check whether the target loss or accuracy values have been reached using the end method + self.save(self.total_epoch,True) # save the model using the save method with total epoch number and True flag as inputs + print('\nSystem have stopped training,Neural network have been saved.') # print a message to indicate the training or online learning has been stopped and the model has been saved + self._time=self.time-int(self.time) # get the fractional part of the time elapsed for this training session + if self._time<0.5: # if the fractional part is less than 0.5 + self.time=int(self.time) # round down the time elapsed to integer + else: # if the fractional part is greater than or equal to 0.5 + self.time=int(self.time)+1 # round up the time elapsed to integer + self.total_time+=self.time # calculate and update the total time elapsed for all trainings + print() # print an empty line for readability + print('epoch:{0}'.format(self.total_epoch)) # print the total epoch number for all trainings + if self.test_flag==False: # if test data is not used + print('last loss:{0:.6f}'.format(self.train_loss)) # print the last train loss value + else: # if test data is used + print('last loss:{0:.6f},last test loss:{1:.6f}'.format(self.train_loss,self.test_loss)) # print the last train loss value and test loss value + try: + if self.nn.accuracy!=None: # if the neural network has an accuracy method defined + if self.acc_flag=='%': # if percentage mode is used for accuracy display + if self.test_flag==False: # if test data is not used + print('last accuracy:{0:.1f}'.format(self.train_acc*100)) # print the last train accuracy value in percentage + else: # if test data is used + print('last accuracy:{0:.1f},last test accuracy:{1:.1f}'.format(self.train_acc*100,self.test_acc*100)) # print the last train accuracy value and test accuracy value in percentage + else: # if decimal mode is used for accuracy display + if self.test_flag==False: # if test data is not used + print('last accuracy:{0:.6f}'.format(self.train_acc)) # print the last train accuracy value in decimal + else: # if test data is used + print('last accuracy:{0:.6f},last test accuracy:{1:.6f}'.format(self.train_acc,self.test_acc)) # print the last train accuracy value and test accuracy value in decimal + except Exception: # if the neural network does not have an accuracy method defined or it fails + pass + print() # print an empty line for readability + print('time:{0}s'.format(self.total_time)) # print the total time elapsed for all trainings + self.stop_flag=True # set the stop flag to True, which means the training or online learning has been stopped + return True # return True to indicate that the training or online learning has been stopped + return False # return False to indicate that the training or online learning has not been stopped + + + def stop_func_(self): # a method to check whether a request to stop the training or online learning has been made or the target loss or accuracy values have been reached + if self.stop==True: # if the stop flag is set to True, it means a request to stop the training or online learning has been made + if self.stop_flag==True or self.stop_func(): # if the stop flag is already True, it means the target loss or accuracy values have been reached, or check whether the target loss or accuracy values have been reached using the stop_func method + return True # return True to indicate that the training or online learning has been stopped + return False # return False to indicate that the training or online learning has not been stopped + + + def visualize_train(self): # a method to visualize the train loss and accuracy values using matplotlib.pyplot + print() # print an empty line for readability + plt.figure(1) # create a new figure with number 1 + plt.plot(np.arange(self.total_epoch),self.train_loss_list) # plot the train loss list with x-axis as the epoch number + plt.title('train loss') # set the title of the plot to 'train loss' + plt.xlabel('epoch') # set the x-axis label to 'epoch' + plt.ylabel('loss') # set the y-axis label to 'loss' + print('train loss:{0:.6f}'.format(self.train_loss)) # print the last train loss value + try: + if self.nn.accuracy!=None: # if the neural network has an accuracy method defined + plt.figure(2) # create a new figure with number 2 + plt.plot(np.arange(self.total_epoch),self.train_acc_list) # plot the train accuracy list with x-axis as the epoch number + plt.title('train acc') # set the title of the plot to 'train acc' + plt.xlabel('epoch') # set the x-axis label to 'epoch' + plt.ylabel('acc') # set the y-axis label to 'acc' + if self.acc_flag=='%': # if percentage mode is used for accuracy display + print('train acc:{0:.1f}'.format(self.train_acc*100)) # print the last train accuracy value in percentage + else: # if decimal mode is used for accuracy display + print('train acc:{0:.6f}'.format(self.train_acc)) # print the last train accuracy value in decimal + except Exception: # if the neural network does not have an accuracy method defined or it fails + pass + return # return nothing for this case \ No newline at end of file From 4c0395bef2dd0e7974ad4d1913ea329e7780e7e1 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Sat, 1 Jul 2023 23:23:34 +0800 Subject: [PATCH 024/337] Delete kernel.txt --- Note 7.0 pv documentation/DL/kernel.txt | 116 ------------------------ 1 file changed, 116 deletions(-) delete mode 100644 Note 7.0 pv documentation/DL/kernel.txt diff --git a/Note 7.0 pv documentation/DL/kernel.txt b/Note 7.0 pv documentation/DL/kernel.txt deleted file mode 100644 index 90b86969..00000000 --- a/Note 7.0 pv documentation/DL/kernel.txt +++ /dev/null @@ -1,116 +0,0 @@ -Use kernel to train neural network. First, you should write a neural network class that include method object and Instance object. -Method object: -Method object need to use Tensorflow 2.0 version and above to write. -1. fp(data) forward propagation function,it receive data,return output. -2. loss(output,labels) loss function,it receive output and labels output loss. -You can also do. -1. fp(data,labels) forward propagation function,it receive data and labels,return output and loss. -2. opt(gradient). - -Instance object: -1.If you don't want to write optimizer by yourself,assign opt as optimizer of tensorflow. -2.Instance include parameter and hyper-parameter,where parameter must be named param, instance can include anything you need. - - ----------------------------------------------------------------------------------------- -If you accomplish your neural network,you can use kernel to train. -example: -import Note.create.DL.kernel as k #import kernel -import tensorflow as tf #import platform -import cnn as c #import neural network -mnist=tf.keras.datasets.mnist -(x_train,y_train),(x_test,y_test)=mnist.load_data() -x_train,x_test =x_train/255.0,x_test/255.0 -y_train=tf.one_hot(y_train,10) -cnn=c.cnn() #create neural network object -kernel=k.kernel(cnn) #start kernel -kernel.platform=tf #use platform -kernel.data(x_train,y_train) #input you data,if you have test data can transfer to kernel API data() - #data can be a list,[data1,data2,...,datan] -kernel.train(32,5) #train neural network - #batch: batch size - #epoch:epoch - - ------------------------------------------------------------------------------------------ -Parallel optimization: -You can use parallel optimization to speed up your training,parallel optimization speed up training by multiprocessing. -Note have three types of parallel optimization: -1. Perform forward propagation and optimization in parallel. (PO1) - -2. Perform forward propagation, one gradient computation or multiple gradient computations and optimization in parallel. (PO2) - -3. Perform forward propagation, gradient computation and optimization in parallel without locks. (PO3) - - ----------------------------------------------------------------------------------------- -Save: -This kernel API can save you neural network and param as a file. -kernel.save(path) - - ----------------------------------------------------------------------------------------- -Restore: -If you want to train your neural network again. -import Note.create.DL.kernel as k -kernel=k.kernel() #don't need neural network object. -kernel.restore(s_path,p_path) #use this API to restore your neural network and param file. - - ------------------------------------------------------------------------------------------ -This kernel API can visualize your taining. -visualize_train(self) -kernel.visualize_train() #use train_visual() by kernel - - ------------------------------------------------------------------------------------------ -Kernel's instance object: -self.nn: neural network object -self.nn.km: kernel mode -self.PO: parallel optimization -self.thread_lock: thread lock -self.thread: thread sum -self.ol: online training function be used for kernel to oniline train -self.suspend: if you want to suspend training,you can assign self.suspend to True -self.stop: if you want to stop training,you can assign self.stop to True -self.save_epoch: if you want to save your neural network,you can assign self.save_epoch to an epoch -self.batch: batch size -self.epoch: epoch -self.train_loss: train loss -self.train_loss_list: self.train_visual() use this instance to visualize train loss's curve -self.test_flag: test flag if you want test,make self.test_flag=True -self.total_epoch: total epoch -self.time: train time -self.total_time:total train time -self.train_data: train data -self.train_labels: train labels -self.nn.param: neural network object's param list -self.gradient: this instance be used for the second kind of parallel optimization,for reduce memory consumption -self.nn.ec: epoch counter -self.nn.bc: batch counter -self.train_counter: count the number of times the train function has been used - - ------------------------------------------------------------------------------------------ -Kernel's methon object: -data(self,train_data,train_labels,test_data=None,test_labels=None): assign data for self.train_data,self.train_labels,self.test_data,self.test_labels -init(self,param=None): initialize some kernel's instance -apply_gradient(self,tape,opt,loss,parameter): optimize neural network -loss_acc(self,output=None,labels_batch=None,loss=None,test_batch=None,total_loss=None,total_acc=None,t=None): compute loss and acc -train(self,batch=None,epoch=None,test_batch=None,file_path=None,one=True,p=None,s=None): train neural network -test(self,test_data,test_labels,batch=None,t=None): output test loss and test acc -suspend_func(self): be used to suspend training -train_visual(self): visualize train loss and train acc by matplotlib -save_p(self,path): save neural network's parameter -save(self,path,i=None,one=True): save neural network and neural network's parameter -restore(self,s_path,p_path): restore neural network and neural network's parameter - - ------------------------------------------------------------------------------------------ -Support suspending training. -Support stop and save in training. -Support stop and save in multithreading training. -Support stop mechanism. -Support gradient attenuation. -Support memory protection mechanism. -Support training priority and memory priority. From 3e4bdb7f3188a8dae9d6d6978d62aa1a58a5bd76 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Sat, 1 Jul 2023 23:25:23 +0800 Subject: [PATCH 025/337] Update and rename reduced kernel/DL/process/kernel_reduced.py to Note 7.0 pv documentation/DL/reduced kernel/process/kernel_reduced.py --- .../DL/reduced kernel}/process/kernel_reduced.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {reduced kernel/DL => Note 7.0 pv documentation/DL/reduced kernel}/process/kernel_reduced.py (100%) diff --git a/reduced kernel/DL/process/kernel_reduced.py b/Note 7.0 pv documentation/DL/reduced kernel/process/kernel_reduced.py similarity index 100% rename from reduced kernel/DL/process/kernel_reduced.py rename to Note 7.0 pv documentation/DL/reduced kernel/process/kernel_reduced.py From 19ab3bac67ddf7f20d7fa51226e90d22ae0b3f55 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Sat, 1 Jul 2023 23:26:10 +0800 Subject: [PATCH 026/337] Update and rename reduced kernel/DL/kernel_reduced.py to Note 7.0 pv documentation/DL/reduced kernel/kernel_reduced.py --- .../DL/reduced kernel}/kernel_reduced.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename {reduced kernel/DL => Note 7.0 pv documentation/DL/reduced kernel}/kernel_reduced.py (98%) diff --git a/reduced kernel/DL/kernel_reduced.py b/Note 7.0 pv documentation/DL/reduced kernel/kernel_reduced.py similarity index 98% rename from reduced kernel/DL/kernel_reduced.py rename to Note 7.0 pv documentation/DL/reduced kernel/kernel_reduced.py index 392d5897..856f99cd 100644 --- a/reduced kernel/DL/kernel_reduced.py +++ b/Note 7.0 pv documentation/DL/reduced kernel/kernel_reduced.py @@ -738,4 +738,4 @@ def visualize_train(self): # a method to visualize the train loss and accuracy print('train acc:{0:.6f}'.format(self.train_acc)) # print the last train accuracy value in decimal except Exception: # if the neural network does not have an accuracy method defined or it fails pass - return # return nothing for this case \ No newline at end of file + return # return nothing for this case From 05984c0dbd86b774a540c4eeca5dbd40442ec29d Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Sat, 1 Jul 2023 23:27:08 +0800 Subject: [PATCH 027/337] Update and rename reduced kernel/RL/nspn/kernel_reduced.py to Note 7.0 pv documentation/RL/reduced kernel/nspn/kernel_reduced.py --- .../RL/reduced kernel}/nspn/kernel_reduced.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {reduced kernel/RL => Note 7.0 pv documentation/RL/reduced kernel}/nspn/kernel_reduced.py (100%) diff --git a/reduced kernel/RL/nspn/kernel_reduced.py b/Note 7.0 pv documentation/RL/reduced kernel/nspn/kernel_reduced.py similarity index 100% rename from reduced kernel/RL/nspn/kernel_reduced.py rename to Note 7.0 pv documentation/RL/reduced kernel/nspn/kernel_reduced.py From 8c049b5a3fb632f3def4882c2272d7523a8cccb5 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Sat, 1 Jul 2023 23:27:39 +0800 Subject: [PATCH 028/337] Update and rename reduced kernel/RL/kernel_reduced.py to Note 7.0 pv documentation/RL/reduced kernel/kernel_reduced.py --- .../RL/reduced kernel}/kernel_reduced.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {reduced kernel/RL => Note 7.0 pv documentation/RL/reduced kernel}/kernel_reduced.py (100%) diff --git a/reduced kernel/RL/kernel_reduced.py b/Note 7.0 pv documentation/RL/reduced kernel/kernel_reduced.py similarity index 100% rename from reduced kernel/RL/kernel_reduced.py rename to Note 7.0 pv documentation/RL/reduced kernel/kernel_reduced.py From 0fd5e2b574360756177d9fdebeb7de05233bc27c Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Sat, 1 Jul 2023 23:28:02 +0800 Subject: [PATCH 029/337] Delete picture/gradient attenuation directory --- picture/gradient attenuation/1/attenuation.png | Bin 13165 -> 0 bytes .../gradient attenuation/1/no attenuation.png | Bin 13372 -> 0 bytes picture/gradient attenuation/2/attenuation.png | Bin 11242 -> 0 bytes .../gradient attenuation/2/no attenuation.png | Bin 12464 -> 0 bytes picture/gradient attenuation/3/attenuation.png | Bin 12469 -> 0 bytes .../gradient attenuation/3/no attenuation.png | Bin 12806 -> 0 bytes 6 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 picture/gradient attenuation/1/attenuation.png delete mode 100644 picture/gradient attenuation/1/no attenuation.png delete mode 100644 picture/gradient attenuation/2/attenuation.png delete mode 100644 picture/gradient attenuation/2/no attenuation.png delete mode 100644 picture/gradient attenuation/3/attenuation.png delete mode 100644 picture/gradient attenuation/3/no attenuation.png diff --git a/picture/gradient attenuation/1/attenuation.png b/picture/gradient attenuation/1/attenuation.png deleted file mode 100644 index 2258c752e8f0d61bf8413e622b5d67066f22151f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 13165 zcmb`uby$?q+CGXRh!Qe%2{M4Fs{|!>$#u%xu1JNlwQgZ-ln;Yg@r{ZCkt1>!nzg% z{>kFuf+sP=&4pN4_ps#P;%e^6TT}RYYG%_ncNpAoaBhe{+qwP#I!#{S(WyYt&6zm2 zZe<{$f22O+IN>@oo%iH;xQvId0-4t%FIO?ejl;&l6~dWd{xJP>)|=0tp|_zZx73^B z&z=eNBsce**QfQJ{_IIkIK4A+`pZeMuc)K5^P!uI3l0RLtg3CLj|G8Ha|Pi+AP?J! z8KKZtDkvQOjY={Qi6k>Y5<_AT1P~}xC7e+L4iAOFkw{tC-@zm463k}5@U*nFejxh3)M*PQ``~HxnK|@zi8H zm(G=g^=u3gX@^0w(0SYBe;oEXtZFp^uI{SWp}kv%J^V-w!jNi%={nHMu;Xs*^UKyt zow$KwB=##yvL=&mw(Hy;e1+e12Lryr3RKC9$#q*f%lsFQ;=;7ui>B{UTq6k`6)GU7 zjm-M}hv@R5RtbCd+oooSZr=T(x-+Fg(Wek<>9E_7?CvdW8~4o=?93}fEfrCVU-jLH zD2Q3XLc@vF6r(VGNav^H7GvKEMHruUpw9(ITk!Po-OU_^HHJR{1j$)k=$A1c`AQ0z z4JgT~KQlrXR4kBcE~?JD=|bMmd;@9+KPlzahbrQUtT#ThLkzc%^| zyX9Kl#CwC`YNbb3`4HxG`AkoglD=P3+ols&XGrxuhRAGV=-8)kXI=#qS5Jm@@q#>3 zDpAgc%s~Zh42eF0iqeSlsRez9eMv}$*99TrcNH=TL3)WH=vs!5JRLKe7w0+a%i+5p zBj?nXiu5Uo`Pv2On@rO2Gx}v3tekHqEoO7RH?Mse@Ti_TxLreRbfexUEI&5O%&iUA zis9qNK)lau>mm1Oft{J{=bmneucG0_s2?Mqw7T`gxe=*7#KdO@iPOZOAfva>4=(Gk2kU{K4vYgJ!_l(7rJB?P{!t1z#JUzr2?3vMp0( zlde&&vyu8n309sy5zHD&Tt*-$D7ZxU+(19S`>msz?c7q)r>NE-RP7XU<&I%WZjH5j z6;(t-N%xLbfuRqrLH>mPrgFWffR%9O{1&<%ZLob|Fl*xR@)AKf5G#h z{?9twWTgXkpV*c6u!+~vW@88`>&_v{uzTW;$N};D4<8)X23QEFxUmK^6$_4CN{WjY z8^xHIB!iUU@EiFCb)J2ma4O{18M>i8f!D(HMZ?tvddYntgb;cW&)Ecv)P8Gy_%PVAY{*SZ*yV3x*vP%=A~^on&!4gUF89+2Rl32Q z+gVPsgYxmIm+Q5h2_Fur^2oLbbQUyRKGKy+=VO+Ta<&f9=mMv*G>Xhpb`A(R3n6&N zD53T%?ec(Ckm-4kyXVCrYGYES=RJwO{9Peecwedr%z0HQU>GJ%@#L2(>=qe!OLCWo2c@t!s7-7vQY!z-c%c-5AplM?okGCMSm8&jibt$!P$9G2mU+#D^! zzIN@}&hc@3C^_4Wb~QeITjmTR1{ z>w=(A;hf?gJp&)Zt1kJiw3G}2fgr=eBoPQ`xy=~VPlGidzs_|_hh4w=Ab*9+pq7gC zzCyIP)4{4THFMBoYj?^~ZC$7pmo5_=?xM^GCm|s*#cx5OD9g*s@$m5_6%-UAd?u%+ zAaryDw6sYy$16)q#sE>`h1~LMIa1M^n>N2a^(RId*(*seeVDNmP_JnY>X69rm~6)P zF)`N|7}ByQ0Q&6Jw+T2c6HHD{a(f-w9H3_y_LqN2f>=p8la`X|%m{CdO-Y%`O>D*T z(iYsAt{*b5BZ9;{RN;enb$6qy`y_fYZ3ExGC!rJdroyM>Y>8o#nVp?gT8R%Orl;pD zV1zz_2>+U!>ZOl5I&6v!+^idw7Uev=Q2F|+Vt9d9wp(dZ#sW(POMzktk-<6P% zSX9UpbXvam-~nDtOw8Uc4nD;_Ha7W4Gvr%md3pILTfi2sh^G2j^ z<_;}eYNj0g`;Uk1FX(S28O>ezR@&zg>-yDyc6kVgb6l7FcB{?VP4Z4uIdam8#>~vE z?=cgUho2zkKJj2o>!|Er2JPr4XSoeWB=T`P@vHQZI+3}B0UA5>%OCr-lu)Qs6Y(pk zHJf~3)LwJUe$|QMI(c3X8rrE*4T7@_+u#IJDO+BM>Zf77O2e2kPw;g@j+BV{oI6UDkHSu)JD8m@9jGKe8R zA{hr1^v8wnl~%AmVGszeZ9-rcsX{IZze5pz)O@JRakthGpXPz$3Osd`x!>>EBT?85!@Af4D$-6*+rl;il_g{v*qV z$Q@^^R?K#>XVF82TsNORdxmv>aZ!Y54XvS9eEz(t(74&i!Xg-D=y#Fd4lYZ7gP))9 z`A)Od#|hK&k=S_t0ZoI#^-L#C%wBYSRwIN!Ko$9)0I)JY?_Xw(LaYsBxYV~L@>vs8 zQDsS2g}%c%`#Cm^8{>u9-ofuyNM>f{4hm#J0hf)fErfz1C_X-(=B%=&215d#JbB_K zA|@hoccQ|sz1CxY=#e($rjT_H%?QrSeFOMj>xNF8B;1rcJ4jYq8rR#~JNvblmsj^0 zjdkaH7Og5r7IqF=PN7GSZU^8JzSngve}X-!cn9r2@*aq64$kGjqH1XJ^xQ)R2w#q$^EoS#mMmP;c$9{-?$ zB667o1*w1h`0?P;BME2c$M;_(4rQ%y8rJIt^#Yo;wBHtaH|$puKW*jw@~-3Si1t5j zaAygc{6)%Fm`gGZruergC&>AsnhE_!lsl3Z7Z;m`nGI*FYn9t1oQrUADgNR)v}&|8 zoMh*04c@gV<9vsdMgKFF^~>^`jap`_sc8FEPTC~*YGY0)+qx2TLC9*WMkWcH=;PqG zoy?8@LfFX7O64FsO1a@`+@Dp2s?^TC&Qz4unlP8h89Xeu4;L%myS=;A7a_(Kgv`n? zL1vk%RUECZtv!g7c*wTEvkNz+p@Kg8QENGzu<7mn!Ce^>?FZo6rU!d^d@jimnI3Bn z(}2XhGD2o`jCYaDef4&D%t3jxS8oK9!py;pHhWlhm1SiMn58g-w~lnR-F8B6g!*&} zp732HV&Z@gUlNWz80;)Dp)rCvVrr2fB<^^5aid+ZtMq1(!nfr`Ys@ABQ6mCi8~GEB zSmb#32p&cPK}xsG{uvUVKAF!X(Cc9v!a#=<6_n;S!&>nBVjk}u$!tuVKWrlIS7O!l z5}s_wPr8%ZT;~5b&aHTYkZmB&C4B*?U#%o#hVoHM!c9c#=wSDMzHog-{*NeE{=f~_ zdGoG%q+{tE>h1dA#fAzJnHb18aGhX#-Tzuj^6vID?}LX+n6GpPL$Qh!&e3frOc(1f zY|dCzG22G-;*fZ($mHhr>(^(|=qPeFjmblI6%~>oLRwrQA)(sKKO5@DnHd>#muF}h zz-j#b{R_>1(eQOYsVowi?5~rgt>R{i3t@$te6d1)jE#jtq1PQ89Ap&~T4FhL7yH)r z^z;h9Bx%Uoka&I7hR&zrU#0 zCp|r#+w%9lDl-cUwrq7aMI1N(umBF|7{DH;&zKkvrn*<~F*TKli>p9dtf8_}3%BE7 zZE&}RRB@=r(A`~t=5Jv&|CRd^KI(V3(jePKD_yCuz)q(|iMOr`FgQ z$&-|oMG^#nAnsbL2ATT*5W{!Qj?wMkUnK2Jxy^hrZ18mqi-=G!@g5eKn3(4e5HjBA z$Cs^9`RpPHhaVRIQ{5Y`CJKNoK+|=*M@*nhcu$6b9Gf7OQv?PrS$|x#-Err31>w zJCfhtk;;ErB#$+cZ~B|ns`0_2jg5tH$Ys8}o#CU`pLWvK;X}pT@bI5a{g;)_7ZjI1 zqopIanV<{p#IWn>ECV%PKh#>PzoG_rF+Dbb5pkpC4+6jagprSx!H}Wv@1=S~0AYfV zEWXX(y6fKm2x_HF1(j~<8o0hMb2^U-w;oM%)W)ovIS2{y@O~y!Sy9$3%8)&pVX1jPy zz@w!FM3kIQ-Ht<($07w82DcL)f)yr-jZ@-CWsJJdV(=j*0j#iyo<7q3bFk0|T5&gp{<%tefJrR?Vi)GPnHp;i_U2iorEW#s_D_6tx9QUq&dKX z2|nI*0XQxZ9ddV}y_@*fty_(U8z|#fV>{i$vB%ZL@zU%OC~)xrb0DLn?B>i1LXJr4jHhVB*Vfi{_V%oX&iGcUmVf>; zDJ^<)lBg75kSF0*E2HD=&ufCJU1MX^N&I$g#b(_=4H=qsYSEH$bmUP{RaI6~3tU~b zR7iaMLm7=qUP%o55#T+$ZWQKLsUQsb0e3*M>s2b1KZ~Rn3H`*TneMdG8!zZ&9?(}* zUfx#ivSE6x2>|^2%aqX1pYOq?rMq5qbalPYi&W3i=rkf}2-7w?6p7mgocEpJu}kar zhcVgO=uChFrnvt~@=51#SjO?rT=U6>;Zd@`dLFk`{q*!ps1l^9~k+K2hr4(o!t`>3ox zKm}U76+C~8QIY=SMjRM@$>)3Tv_MfWkc?sTrv79V@W9Rwtlbeh`NqEat2x#&wKXdQ zsWgjBs8J|%Anrbca*JHh{8MOIMc3I6vmaXbrx`FjfBP@bj|q9@C=X{pec5K_-AA||JsG#QaMZS0E=0=~RXyp1a z5XDFU;Wf6l4e?;AnV(Hy6Y##w#sE)<5i&mJRe(^Zx-XNPld&{(;7$;d+My6s{d4*| zFXpAb>CNA`1+@|lLMC{4Rth*xk;HiQ{)GlC@W{cTP)f_5)!Ty^o4Sf0QyYZSNwyG# zTyveP^RbB_R^I1^_e8r1c`2-~yju-?tk=nffp{K3#&}s?iq>lDg%bXQqK3z>GZve) z6WzFRDR!E z=f_00yGDIPqxQ4xdv>2NaZ3I0f6yrCbYDluPjE3ER(~_n3ArlN zYdAZXCN3o(SfePLOdl$I710)epFWP06~EU3y! zA^>Ej#pPuvFR!YohKh&KKTM^d2x0AX(f4x+Pdb{=QF(I)zk4+$hjO4peJHXr`tJ z-=;?Xvmn3P3PIAO14^PH1(5IZ_~786ruy6ZgaybLtK(%k1UR&`w8DTEA)}%+^ITQc z9r_=MCHMBWSFmzo(keBzx+JrJfEH1F*a%!%IfjlZ?eQy>2%na%2wMIwiq6J>n`RZC zR_jTowC2mH@F7uK<7N17-@dhqwJVnJ^7#BAOk{f6r@A-x9z@v(e2cZUb*^p&5hEib z4;BEwN12<_80^C%B>aSpMqph69kFr3u7Ly1z+;LBg54ivwa;$o=?83VAEq>hyyvSo z{#ihzoyWJ`V@#JW`FCOzuz|1%q|wmQ3!ORNzekkT*NQDNxw~#sH@h^cvr&wc#Rd?# z7w+<2XH09Z1gCj-zU5g0fnEzIuHiwxKKb#Oxq zUHb<#{j)tO1Ft~jdK>YVnv=>=lZE6)_l3@SffOI?FV@G#hrOzm>MGeTui!&X-0-ZI zh;{Vh+v(BMO-tdSZdFQXr@Q;sos%`Xg1(n6X4JidWZ`nx+3`RY7QmKIfe=o59X9Un}BghC@=@T~svQSF6KGG250IL$@{ zXFf;4Vu=1&roVJ=04d8v0I@25o1$On)9<6jlTSMzX@*G7SKn(@Toaay=^%i?Wib>= zAL)#G*C~5a?)K9u?QUz&9g=9*jAt8Q0x2*7g~Pj+K1zaiLk7a8IVVln<*&NEKg=1J z3E08<$JjYIxSg~p1$RV;P&nP?X+KHr(Cr~T!g=ycP;3)q^t7|{>xet4M6s&=g9Xzq zOm-JcTYAJv~}X7 z;GRvyi~%JTWo==1j$0c1_^)qm+QR$>6;GyQ#O4MLCBPcf%@N3Gt3Q7P7ILnt@A%=} z`q@hw$GsgP0tB*s%ts=@E~#TG$;uf9hcKD+;7q2=lh%ThUt(7QC4#3qankVH+;CY^ z-`BRp7d=YnGYfSN>`<#W*^DZn*MLOgwY0VZksqR4ZWAMxLmI`x%G%8VXc_>UTu+Z`u|j92vaNdx*7GUtcZPUCo~MMa}D6Q&eHL3Q;4B2@Je3 z)&$x-@AC3EKv#m8lvLG(!NF^}C&BSF{EK98*NBX{`Tf1Uy+Sv%0j8?jX2mGLx#p+e z(&v1)icC5mu+uFBa}in0zE$@syoghHdGo%NF_<|#oB-1#nL0W;iaq8tPvf=d{rT$` zVTnbb^ws4B%x+qEaRnv9p;l^yOGv|90aWE?Zz`TjjVBr>3xV{bf6gE~_QjGd?ZLY} zZE4XSTi-`AApN#LZPK5CN`-xeoqhcHv11oxUYErVh=72=kl=S{;6K$P(niTPj*jkI zhL^8zT+)qWi+wX)XjJzqTALO8xSPlVPgLcD!*ie<9C_=mo}N!@YinuBqACq-B!dWI zj>E1Utp{wIj_2mFLqxmo7;;g+s~3HFGQ$-$_ah5iw#r!&1Yb{LHX{ig9sA85)&C%x zZKCr>Z4tC)VT-ek0U&$$XJ)d3KAkB)3NVi)-x}8~3D6E1W>3?3)0`=SA_0Z;u zPqlY_eWkgRLJdN9rs}w88g56zHQ$y;0_rt#dHU5zP;TqQS*ZMr$NraC#W;AFw|$SX zx`NYyt7UUee|Pd^+rCis@C;IhmCjdv& zS8jzKicDXRKZAMlx9T0+<-7~U)C~$2?V^s~bCC1c@(94gBvB+6(#Pmj zO)Y4Er_KKx;CYige+`cv;(Q>D| zo8+}Sty}^rDN&%l{&@2?0VW6r#YxoGOZy?$oNr`yettCQ zO;L`TvX*c$2xpl^NMg-*I(}-XspIXi*0)d`^rm1$d7jPA0F%g)%Z$foZUd-5%fa?m zgXz?*NR5hsuWrqa%9OAi#nR`XIx(`>q^wFTZSbPS_PxH)MEoO&5mEorT)&;cCGUOO zpd%2Z2)pSwaL}$t00bkLbqK|nHJ#N%911on0!+LW@92j$V>6v6QP7oz?1zhF_hihux-ew=fU;!NjlRUQ)Iy!Jt=JfqY9y9VBkRx z&PkddLvyO?m?O>34C}G@8yU>jsoiL=R~1C&**HvQQ_%T1H_l~V&1Q9Pe~8;@fVD@v zF5L$lIza5*(VLPwkH}0IMmJYtaky^X(?_{j#WG*}_TASD&}&SaqNE{S&^UX~=5MmH zHEVBW8?cMe_!QcxoU}TkQ@24jJrNh`di2B*5hVDpX>8-{CYy@4Sa;A&)H}B5dwooJ ztG_8kgeSTh)q_Hfk+Y*Y*0;K51svpXtMDSDKZmYWHuK$%tfp(QC-l2S6$m8y5`L|& zF&_I`_q%i0yKRsPIt=onX!_awFFVCK__7PxRDF33728@iek?t`?GnZ8+yg()3Ts=} z8R^3>JOVAVC;1bNb2f(k^YNqY@|1M>g+*4>V98X96V9Iz!g*?_P<@_!q2*~*S?frS zA_1n8;LL>x2?yy%wyT=c!kNWvsdENoS8d1oh!nnRH%lf-79|uShJ$hnJrC7U`+yfI zQlU#X9KiG~Gk7!GsuSEMhqbqHh{*1e*;dNiPFe=Re^2ORwSg8pr3cv;6$ocF2d`r4 zSEbIuAD`$xQ|f-^RBKTLIuXZrU1<)efNNSVL1b75*P~A+w;9Fhi*u~vV{l~jUFo`; zx^<(vmt4#VH=uDS$O>Zul+#P%+s%0_SfP_pl{#odX?whL)8R-U&!cbvn#QZv!xy<6 z9J3qC_jDg!tEg%9bbs0;Pbu(yh(&{%Ev( zOZbdAoBzUlPJ!ZVe@S*WCfs(`qoeoM@!CmB_CqTjr#UBs4ZfeQ+7R9{fgzcy)_L6n zw5}HF&BTlxOCg*^UHOGpQwUU!ONfi}YyDvs4cKS&16>}DCEeQiBOM4`HHFLo3z-^@ zOzW*T%OoJcxQNhK(V@daifcM)v06{-g?z1{ti?QkZ?HU5bqh}cxd!2^CTN9wzGSK3 zI<+=2V{F&R?jt5ujTWIBUL(oDpvvsYrK^2q=3r$>d0$*Z*S-ND zFD9o8b^U|g${WRr@BSWS_&Y7r9fe*(>IaDN-OdS)IQ;6LKbU(L^J%Pib1kc!(Z4mg zuxp_L^#wK#$(4agtr?ax%P+%BxC6c#%!j?zda_ht$O1h!FX}hM0{vwWyx~(o3s0`q zf;<}5<(y|nKc5$l?SfGV5^5i-CZin-0(`}lL-I3}VL8v8)ravSMblg7tVpJKtvWd# zH0_Ntdpkb(mBsa@6$4ii-(|h`$penmRijECNh~Y4`l`pp=W#wepPcA;R>iwKkJmmj z!Xr?A zSIJemF0X!J+D=&iO|#q8a&#=71X6dQ{ah1GRDpI0?%&et1^gT3Yul|tR~U1x&u_rte;CbeF4n$0v$?pyt2+2KznTWKzKGf*a5n*^ z=+LL)vdo&{iayXN00yGSx7atQ_kEh0o0mREgCaH@4QGJzFo+wxJVBwQXP0C==QB$* zxqT1m;oE{7sAV2pOb1^5ZwH=YO4Y!?AY$ACkiOAs7q)8WHE2y^{n@qi<0-mf1^yErw6Rx%6kjodjA7PWM+o<4khwh5P^OTF*Wr=2E#(5YZ8)@5L#Nco3lU?E%LqaPQC0+8Ztf`dT*0yj0dre;6;GU!&JP z#*S@IiKZkQ1jaCA2NJozOuvPK)TGizbYef9pAxYExz+3CD<=dW=Kt{oCngX}AM24s z43BDR%zdKYNa;r2JNv_-zF`Ec#MDJ!>hqDh!X#k~tM@anhq+XWp6=FKZP@>Px84#? zA;QHN${E1U`G!RUkMqGJ*ooJER}dV2UqTNC`;+ix@48lS-%```$WE|3Fd@g>`caD& zJF4a(rat`+iBuo~YxsXRE&sz-qi|^|>F~nU;S(oIaY@PC^78kUP@N~f=7Scm(==x- zUeh>C+OU7P6O)`w*V@O|F2AwIndD=>(KE zNYvW?B@Yo>BO1XgP?b$S%4mo^ScV!+1X9s;DLGWZ}YkU8Gf|hM`-z1Ql`tp z`yq@fVFJ;A%Q}iYh^km+yiid~Gce|Gn|2WA$-@Nt2PpLE(l;=jipODnwWggUxq4M3 zv9Zfem86D5ZBYzdx}(a<%D6sU`=>ci-dpaW%!y`3dWL~g#qF_Y3Y;-Ck&%&@@6gcD zI5}NOTdHvCUw?9#FW>LNa(!^J7c$o1Cq~2|8eUl`aK6BB6+JZ^1UR--sc_3@YnENH zq`0`aAo7?m$Za~vJzzU*Csh>w;v$&s=<3=Q4>MV0CQ-dl^c{EwKAGG|xBgRuu5S*f z<}Kyyo_!Rh4SmMockXcdu2G2!T84aku()XY?fduitSmg>5R8BP>ivjWOlkmjB=?*2=+-=VJ%g_ z0eW8{-l*_+feh&ND&d%H`QLv}@L!HMj-lm9pS&fGXT%6?mOU?3jB~yE(ESr z@2ZY3S{%n!E2$sYaKv_gV(-~Fi0A6E#2Qeto`)M$ijhyoHz-%v4N^ zJq|0Y11cf3kDao+WT>p1HYOJQ%;6WNrR}}f$qF(Kgc$L*IZE`8EGIu>(=&E|e$NBT z+Py9|M?Z(USd5x~f|2&60v@3$uK}ye+pjWFP*G6>PIWHi@VnOe=#Y>QU^EN&W=M~1 z(b>3;3?Iwauh|(?6zepD-zd8ON$cPxyU^v3s$%C^i@zV@ZM!upzq2d{o`qLp{;TPO zB9OZTN#baUIY_^N){c4smyQ5}nnt|TxrIgvA%~wH>(3}LM3C+!7x-3dwV+zl(5I|z zD~OZzw-V*2lob78bowV%;NdM)fi@etQ7H2wcOwBh(E&5c`-%d+Du2*d`jVTgFwNzC zzS~}Wi1F;YZjBqy)cfrD7GZs}{DjVky-A)MJ6ISAnE4BC{sh= zCaDcg#v=kMum|Mc&6_vr#C-J(2;>5TsrVsJY)0Pzds#;`ah#DsgKu4};f+wLk5}D> zmr%n6a6F~I}{KF1eESvAdS+EARUYD?rx;vjP<_n zxA(jE`Sy3t`Eh>eb*;smV~#QAc<$%E@8?;cloX_HW0PV-5OiDm#dBo{LW%(Yq%hIJ zH^wDt8W8jVl79YF)irT@8cS8xtekDHCj|5Tg9j0$$dCNE=sBiav84M(LQqpI?~^2lEiF|{2uj7v&C6RJ(VKUMAm969vEcui zJG9_?K7T9-TDxtC3_0cXj5wT#WLR^om?1%9#65=g zhWNtYCg$VaeFPa8r(A8X+dY0|KeV*y$o?D=a>9W=%798W<@nQeH={*hW6oBW`--f? zy|-}GiQ7wW^|#~^TWIKoF6!~wMIMSV5ONDIg68HFYlR@qq|HB==zBv(|z*xW*a`3d7L?DhU80P z(NFDT6ui;p5&Wj@b-;nPs&E7$=f&s2DTQH?vH)6ZRbk8alYY~r?&u-b8`{z{-8UL^ zTx9|DWv~nNt)RSCiMZ1VXte+Gl#f&er2DkCEM?KHtDx=rIH8J zZk4Y8^jMC@-K9yfCa){kRH>P~v2oPQo?dlYx7IoRD)Q>;t}>PNiIeUz zuE&5XUzXR(FP<|?_Tl;}t+s$R$E9KKVfdHYDqVtK^=pl}`gaf~wi{P0t<3UkvU_COElAq-0|sISZAh&df$#1~WNs4q*LxB|^b$oJ_%oX2 z!&S|uG{eA6mIUi zv#A^#S`^*rd$4`%Ea){{ zeM_BkbN_Kkvg{)@QU9-IPE+-znZifk1kBa+XC}4PW+3!);v{&A+{w#ggA;w;rz@k&0Bt&s>7kRR#R==(9Ro%(% zWMKJD3L4bOIbl`kA^3F&x5VjGwtjh$LRZ_Ou%@xkG)b@Y!&qmOj!X` zI7l<{sECLAs|;lpn#i0n2c^wddlN#j7(EenJMukS&ZMSADuUYCcI638jDvDDUhthP zuixw@WpI_}H7T#%i#~5xY|iXHJ>ISkJQ=i2y8IlmbdtV*eBfcjCs7{AFe!ZPUFJqb zA(D4>s5>!zE+TyGWt=3KC8Cu+UC;3PT-{A6if#vv>vwvR{{IIzSaE>LXurOM?VGKD=*(A)Kz4>*!EyjbZbLj$Y5!5E3u%Fc$4{ z;$jJC1w$Ag66WFMWt>KZS{}+l?t7d1H}T%*>vZSCMcxuSJ3FO@&FIyhXPo2Xguw-T8!|k<02neU!T&!-d^g(PEDA|=^Tpqbvy)iPk6k_Ase-I_WCzpZca|xaGrWz z%EkF+(Rz0r*U?GCjqZ-9mFAZ7it5wB_DLsrpVZ8DI80X9#+)!SGiQ?V*}SkQ&~NZ|y!dUc_5C#(xTC*7 zyJBtZyAii(H?gn}7eoOz?%aI3+G(i4$7kHIL?W25gU^SEh{$bcC`V;OPuxsFLE)&A zsi@srS^4XcdvB89=%$%{*z4h3wSlg=(c1l1vdH9QRv5208}64|M6BU{ekh-*M3d2R zh#wS{R8(Y7P3hzo6htK}IXafkd*6lp*^$;t!o%+z!{z1WOHF%7t8cC^OfY)=es7&0 zZ<6YF!RV#hQdS!(tE!~gGWgVs^rG&mlU>L8`FCmRe3g}x%Z!X9z{bIe)Ke7~_d}hL zqVl2^zK{D=v&6K2yu|FHau|w7oQ9u^kL!f5=L4yfB_+`*p1c)ve=Jnkq@|^WGeL;^ ziAlZi+wP$&Zs6C3hK3`#kwRTrH#axs&hv#}HuJVniXW7L-PczaIhmRMk|D%TWuuun z%xe(uu5ONkCrt6kzwJ?FZHsWIpvz8Q!_7ql8I0v{r6)u5YF$xPN$G(w$#r6OinkVW zK-)|2%YCVJ8oS+(%>iAyQaK5B%to)eD&2I@r9?jX6cUmdNa_B< z)MX!sJ(<6?j?~uHHtxyZPR%YU%*&u=OABWj4+DY{`guE0LC8RTWKF0Lbo7B834-dt zPC;U}ye=biPyNty@aRz>XnXIw$~1+4mldnrlNSn!PtqK@Ac$GU%E~GXp3#l=f1*x* zqCx$pMUyi+eDw&ak@2CNW1|V9x(CL1N4%D}ibJ;20QswpRqfH2MU%~z zJ41ZR%Sna1KH2r3M0yy~ed>fLXau0OyG_5IJ>ehgTG)AAEvnht@!DPFiaLooH{DJg zJ$}mpsqw&om&nml#-EBm*PZ%8A%=090AhEby%kaCaiaYEIch*aKtDM2HA3*1bp7wI z1B6814AFOXbgb`rs;J<9rVxxag(nwy=clK?zjyB*y^xTO4HFa7=X#3rUnME{ErI8K z*R_^1d}vU2qI8qLpTXI|a%5DL1USj1mZPjy)?q1KaCi(23(Mzd7VVMwGx-GGm&V4% zfo)GBvlLTUi{`cEAEKSdk!&jxb=l@clfn{^A=7SZ!>>t6EW)7xyzHHwvp~3Ywzsbp z}M($FyNTqLFd83mhn>js#Nx`6N5CaK?4wGGuWBc-Xu$~GyHe2DY<`5-bO4ZS~}F%!O=0ZCxK7p_3QNREv-`XyAo1TgW!!V zoxx;dl$Dk5nV6b(iq$0~kXcWb7mS|hmYCkAp`#o8E2ouuoY)xI2FWkjKaQz(_I3z( zv%?#I)F=%V<2bX#qci0b<24HnW-_i&Z>9 zrYz<3UK2am*$q)9g3JL-7d#Od8!Wx+?YL$Ym*{v8N)`6UGPgF36rIku%-nj9{l{rR zH$>K2+>GCIQdHVN_U*kxn_A}Gi+hjW^UZy-gbF06PAt|}p>oKntU6Rdzb?m71!@d= zASXsQ`1tA*TlGiqsM0`oq@=y0xiQ0;QarP}=)TTi5H41yq&nwd?`k`#^(@FY4vf-k zDtSA4(yJ2rF;w~teEe`)+5hn;Wu*Zj^zETss$%8P@%A!yZl!J!;vqN>z*qOu(N#)x z7E$Z)A+cC7sutASsgR<2WHls)`0FVtoz8~9SAB)DV|f`q42a+Z`#X_EWp!O~uB*c) zC&cow#bd=%i z+yf@3M;Ww7V}=19-8k}4PK<4bHI$p7TT0>bqh%sA#_Er?_E}a}RBbBl5XOUuUxL>^ z*vD&J*d(4mSJKcR21&cV>+rI`xh3fS!R5D?8T52?Qj|q`c^O#vw0+>%G6)F7gPfJ; z-BazfNl#Dz5m2MYvZkh4IlLGXad(%coGV+geC4^MW4U>F`VW@77?_xX4-O8@M+WH`}(-uah-mnuuq!-@iCM_ew*|H)= z=S^A2P5bNC=o6%3Lt3netu3o{JO2+KzLu15!7?+IBqb#SoldXcqup9+-#9&W1sXE_ z>(_f#yE{9qo7~*o?M-3t+~$=>K{{|O%YB7-r^s^mXj@uZ2=3iWucsm>XAZNd!9B66 z3|R*8jD>~8ZMX3J*gb*IHu}d{xL_ z4I;o;@up{vJGw`up?-dTk>3F_WyVA#0Nv%lu(i+pAa3L^U(N(4SLFi7p)baiviN05jG|&Tzr%{7M&R!5*mrSvdZEUh z^>v22+pMztr&xIf1uzs0+`f-@A1{f7<&>UnmB8s~XrP^Mo1sLGGbpKk&sS{G#U0IO?6g6V?2|FRH4tv0}rc8IRq#*AA&69M{Za)>>A=XVm|y7s}3=YFQ1zSpeSOjsLh zAwwKrIZQ)j^y9`Wk)hdfq>^gL3a1Gbr*pXDUDoqca0C}g=^$Pda7kN9b*Jq#EJ8x< zkGoL8SA1&2Ty~`u;yJe(KRl3YoD-T(q!$^&060+m5EGvro7G8|xuHPo?Pq*Q+m{_F zV~*XB-U(?8-5-lsB2`_Klw|3XGLi3RfDip|cb6`j7#VBSvJn8#euvh8+w(KNh$j3e z7a~&2K&19RKlM=h=pqUrzzS+Y4fz0H=^lN;@4k;dJY}s~%?}wo2M=Ww**Ees;-cUW z>!Lx?-~x{tK21ZhNKL>8x$G6LA+qH1nt>ufu=T!i&Nt?7ULkIz1~&?@I{ghBCEtVj zUYJy}RSAqQn!7?wfO2JQu5LYe*R)8>>l|ecCA?!tikMR#9o`TS@H);y~sHm(7@->t{F&3L?haSO8*g=K^tb;|rKJmwoAM|W&Vb0qV zG!@(X4rga)KQdvk{vRe?OLjmT2sp0YNfK}hQb-nJNOQbAwQ~yUpqKOn6iCQvpI znJBb-4_?YKhjgs5nVD?ZV{Y!q#H1t!8kzu3PEJJ1Bx2Thu<8)>PbP5zJ*|dWg_Mildg}rqJ#isp&JN0KA03 z?*9Gz(rk2meCpY7MR|D`KAl4Q!sce)pykivQ|6|gTv47BJv1mk5RecL5O2N=`8ja@TVO&a`)lnWkJ{5a=&Yomp?PTo|)6Y}0)lN*e zNjUwTcW11FdXR+eaxB~U)Ynpo3iHfdk$g=`EWdw`pxJ*Q6qe*`f2w-4s5hyICxw^A zmqJ%Ju4C(p2;B>5-2Z#9ZBMCPH;Acj|-X^3sj7b ziLdla{BHeH{zjU?D5^M0$Kq{^ssw>tse$B=MgKuoSKa0xx$+25la}s} zwJzZdARMFd43UHO$>wB?0$rtZZb8gI-4L|E3aNr}b26*{NHmmyPkbwXJcl<2aEK7N zzd_iCfZ@+x1M1)#08n88P=}rtGF9ElRDc|*Vc_~FtWG?<6@Bv#h&&1d8GWJkgKPf- z7a|ao%*5akX14IrC;J)AW?gGGj|=UP4L$+9A#|Nua`uV{NrHTrpT_!Xy3To1ZR~FO zf?x;$V`#p0vJdCN+Eovb=0e%2!9;9Xc8jh3^F!Jcwzy3_$2=B#;1EN8hYken(led;`?Q#AMnKn!!6TT)Cy9v8m~R zt@j}rsjxeDp%fDO{QAmseDONi5E>LMvC2*&>Mhih$gk2w&B5`-X1ZG9_iti%q-KCZ z0Y6BqH9`@aelJPIU~{CP)ceZ){P(17M^DdZQc}{)q2Yvo3YU&e!Nr-Lp!-ZqXD>4( zUn(es3h8NRyimzjfVH>30Jsy)k-!TE$w*CoK+0tr!c6|wV0;2FpcyP*lYV}=`m+@> zEHpMYHusyWll9|KeL6)&IHmsJ(9rX)7`E*EeCE!st_xLQ-ib4At4X*;Uh@G&2>SAanucb*n%}?_$cC44eULwvZvc3l zD;-?;E0|pWhudW4Ykg^@20!0Z>PP(dLTB~ldr3*gKWPj$CT2K^QFn7QNaKLARP&NC zW5!h5d*^2U&~`$kveHb4*td%BTuy5beXh?|P3G+)lJ|N=si~iPi9UH!Df_^@q!)NL z>leEX*`M&~a;YY3-AW=`2wAkA12C?(!Hs>$>a?F2>Sy*Aun^faFxu|iXTF)#Bte&f ze9e+f|CU8s;ePJPU=-;Xr#EljtbL+OPW?>A*EwX|T9*lqj?AZE4|cZt)AUyO>JLpT zt9)jmB$Dn)0Gk90(|AB>+)8(d^*rjAT3T3m)RurlDWu_!$5wZw99A^1hmMz7!NACv z-q+w7?bPEY?#}5<5OV27Zx(!%Fu$ zP?WRZjRKF9TvL-Uj;kvAoil~QqVl!Ma+@t(DS z4dkTerNuDOECd(cnXXwjqii@`icoa-yd74uH-=6X)07sTn8-ZnSz$H)=+ z<#?*g^<>&*c|SucOh$UUFHKxG3o6p9i!KDAS2R z&=lWsW)YyQ@0F_Eooz_Dyu1Wt?P+%`hq$SWCNd@8yggBDgU_T_(C?&r$=NF@WPbj+ zMh%F@sbY7D@&4eMkGX?GiNK54u&^*8bOUppjcU!&7qf(_hQ^*4>WE3) z?yf*B;p3V}>D8-|2)OSOX@d{Y|CVG>oOOIV7u9+F!75qJ(=#VX1?2Q>gyeV^#w}KQQX#kVN21$Bz-xjn{f2 z(_3Cb0*!{6TFKS5a_ZHq*B`Z=7BWiD@Q^nO<(`##mCdw>h zrpnvkKp=H}jgFS=ie`yuqoANzFQ}MF=Q8W<+h1&3PjZ=g*%1AAI+k>5?&=Hy30z|E zmA#|4H=0t)+?<|YP*6oi2HV8g7@<(uG>XM~e-cnpDXOWdsVXZ!LqbAA7|)u4MdEPb zNZ>gi|0^joKySL7Vi>9r1u59HiQ2G9@CM=OUIE+O>A~8Mzk^-p)xdx<7Fp{bY)tVV zY^>|{5BXFfRwplqeRQ=^Uc^Qxn0;mMcNBt7#2ht&m-8XnbpdOl((dtXVs^3rjgNWS z<>Y;@3cyWi=23tmO%d+SzrdJhRdhsP!$nVMRCB)VW=-;K>p24Uf(mF~DfLyGzaqc+_2v^}w!Ka8EzYoH@dK&3KQ4R~FDcXW`r z4F9A9MewM>X-Oda#iVJm@RPrk{l&?YUMMI;ThwcpTYU+jp`q!3gz<*hG=KQdc%A6F zT>?R0>aePSpw*vAxFn;yyXD+$dg6IXeQvztNV_XMsjP0rR0CJ1a|{cjZ!2tRRe1Z$ zZ%@b>c_ghc?Ne)Ac?u;3+4Gy@u@|re6oazrNtajZ>Rl|Mz8JwPz*Z{heUWvza z+5sI#1Hy>0`{SFMRxEcqNCrf;8s@=<6vtB(5W-5BO{n?kAUUF$MukEU{t2QN03^x- zZI>t;Y~LP0i-rLP96qeXfTy}38@vIdlRifOFI7eWG+y4%80Hrr$>nZp-zM42{vpeF z(F~DanV6P}{+R_TF-d(H9?CW5{`v<}vvYv3yf1~}5zpI{f}6yW@gd6;u^C)|mN@|= z91XZBPkjbR{}>LqFsWm{erdxv48R8#7-!?i!8ujm|Ex1d&1!4H777BhR9jG1w^L(D zs&`jG+=UA6f~`NkY@Z250M%Xak%r2JD2QU?a|tF&EMnD!G1a}1Q<+Ucf&lFrH^8`W zXiD`KQ9<%uiGjqsDk^bzmTe4Hh>lSP`@1bumBO@X*ylwi^B@kvN z$axfcB{tIn*x1t z_JJkf=cGo^DRzF1j5Hnsqcy_{fr*z}`-fkaA1~cREhwn%PM(&QwzRb+$H2f4`Q=ML zA_xqABJ&?wdU^^0YZ_y*Fq*|@&DEX+DMV>;bF@enZ)I3rAJ!5~G`KOG2m93m3jrn0evjQc zGa$TfEpD|V#D$+(=eL}y0P-yl))Zvloh>b^t>Hv%y#zxUQD9b@MNB)TyTGxA1i5ExvpL3>eT zF7yrglHiPMQ21kwh4{CR;ESdMIq?=mCW$wSj)mMuB7|B&1)mv!VXCGoPcvL5!`H$b zIh~);K*^HI=0l$*c>Ec7sLZj_pSRijaFlp|5=sFTIw}tUe`#|(!ufM z(f+4r=Sww@5UAxTtV8c)PcO_b65FYEQsT{=mOo`#dUC|W-xrasLRW^~1*y})67Woc zO~Xs~a3ZDrI>R^DlB~T1bOunLEk%2%A$_^tR&=~n4j{i0Qh6#h)lvSh{?=gVKQq zHel3oxgW2C-}>_SGPC$8wB~ZpQmM<1>9cslN3*J_e=}y|85;pJ>^e?{qP= z-;kLsj3MECY{CXgcTs~RaJ(Wae>X7Igx6QO`T-*l9D<~%>D&y(2Bv7`S3i#@m*y8< zD7K&v4t^J>r!^NVB8ncwXmF7W&t|^i<$B*6HS<+z-hPaUB->(pHJicaTg8zE<*VZm zWT6X^0u}=i!~wOPA9;IS&qV4`Hq0^9O_7~W53#S`J#ygn!VC5@ z`89|1BVUGC;Vjo+p2ScG+vC{Q?MDdNf>TjjI(GEjI~!Wt$Fb2q{1~S7 z)2_i|QbqQ1CO+@v0d4og!Vq~O4v~*vjl_)!qpPYl^d56HJ<+ISZk5FF$aQB%iX^}n z6ZgEs9F1y33r)`{6UCT$(-%e$=EveE4_urKj?yQ|+Qr!1F`HIZ!`l#D{m>pzKaq75 z&xKpPoU4b=M*eZiXkK~jJip|}TmGH9YTx!E7nQhhCHioiP_0HvF@55`{G#*bj$!Xm z@$y?8*YHr#iyGH_ICHf&Yl@pq*B01kpx8w*L=*jo2lcbPe=(6kD|9+P2~qiauu(<$Ry)vF4rOLz#lIIhgq9{B-hJ zwc}NJqMTiCteCq1ufHUk$7{3kARk3?X0VUzlb)x&HWu>*NhJ4Tx;a$Ao(tpj_=J*CxML!{b4Ml zhO3NYc&=1)gCM2Z{5~sc@P1l1T3@h#`_Iz-Ikz+3ZnFWN$P}BYmx}9k&LyFq=YoR^ z`cqPhIqnZVf4}Yv4Xe2YlOuj4yS(i_;gh9m8gO+yW^&$eL!T<9sP0}_*2ZbmE|y1W z#A!yGpkL{-s~7a~V`_0RyH?UIi z3ezNGDVHniM?3Sk2uS9hYA2IO&DIJf=_v#~4_`A(Weh(%jkj)y167_BF&{Zf`IqmK zkFNKbj+UZr>90TgmGVB03DvJot5>)c`$4)>2gnj`&(pUP4L+i17#J!#I^^3EWx?s` z=|h_c^J&tyfu-W+e6(Q=6jx)kotN70+Pk=9qunA@vau;V=gCtqq6Iw)TEKEdB?#=G zy50+Sh z(SmAphKC1p!uqGd@RzL^E+XWMY$&ImS&-DNC%kQmaT0M%|HWqPMgQ`h{jp|Oi{~cH z{=sChK+U-iSVrrhK&CqrqwjFKXL#l|-xR=Y(iy4OjGD@I&DWL<^SRMiU;8~Rxly!y zAKeY3)ogwFr(-=qamgFAmb%2q%7@(R-kZq-mNr5 zE-x>8&EOX6kt6%?Yl%tq#&mJuPS5M%P~kl^Tfa*4e?u0(r14SRRI1;|$S)HIrE`G| zr9=v?Q(1fVxqb>;2AkT?rnZB<4&k%+=AazEUjNkQ-ZMa6?7O%&Pgju&!m9EJ{X>5v zs%MNANoM-yB?|g5P(^R5%tP#W_`~ z4ZSlYzdv@svix)@M(7FrR+!m-OaOD$`z4+#a}8Uaq%A?zP$|o~BLXqeq9QGw#3f;? z`Psi&=l{ah@?;G?yZnwIub;<9fM>5a#wml+O0;dJkZ+eD)lR_Pq7clbmeG+{$W63I z87;JYnVlW4PysiWa?L=KSifr}no;SEY8Q(pDYe0yotcZOZH{`Ml4$|Yw2PdRg2I494y> zz+k&vv11wm6ZBxVE2}fT_`S^u!T)gMS4@4QfOSHp)E-m4d?>YjUHDNZcQbS5IsqdP zg#N=PwvqSeje>QiD2}S1QTxoV5sQ#Zo_N=f6H0Ese_^AxdHC~(nfC7CrbmKXD+L*D zWMN&32NI8%xNVL&Ue2+ZQS>%=HKC!Q_Xs|H`ovihoP93&H--sX1HN`IE&RjSKR*UN zND=RT{|mvC{Yd?(On#xHa=mqFa?%_n9T| z=&IxjCkQ45iCFEcGLM-~`dXLYZXH`GAIwSpTmKUINDloprYxuTr7JAK!2Yl6KFqAt zxa@Yaw1UQ{R5-gSXcb5I`>R!|wDeD84)4M)(r?fMe2hY0YaDb4zJC2W(k4infRJzi zbj>kvaw@Yj)L9JD!V-)fot&Jw4VzGb8((U52rQf^o39;v%gvz2Ot!=f{;HaRmcwd{ zos{368fgD4c-#0fx)H6`oR4dBAYS&>#Yq>>+Ty|jqHs@e|31udFl43u%j2b;9fezj z%&DNc4whi#2rLD_W%xpYoloNz>G6B(`$hxD&1i$yMfqhC87b)+QNw9~zqzMp?SMR2 zwdnPJn0;*vIN0mKZ1qaj{E}^kO_PG=eGmK^_*&&0!x5(V*MI5$QL5II;f?O<`P^62XV2ctTOD>rFW(N%mu7b8;{Sq)(fWj8n)YLrc72JJ6GT8^(&E}FZ@%qZ=Z(KqB4!ayP~pD zQCJ5NhPZgF`8$lM!^6Xzf02{mx<=p1lPcH!*Cb4uWpAPedN{B|{H)Z}qAqo`wPi7I z$y4t>mZYCzTy$Uj%(v9_AcphoX1~;8aP8`3RvqknO9Zp4vh*Yz;0{+Ov%LQ&LUIZU z3lQm%v9a`m382g$Cu7mG0>mYr&>jQ|1`6(`+7&ix)zXqJvl=ctQw+1JZaD8^fU5*c zFi{oq_?#^sM#>{o&%Ph~sX#8Oj0OgSajJx6`yQ(9A-X7=@kFim;6H~9z;7kE zcKAL0k3W{s2nXlX=$ti{=D(Va4NyH*d8i(=Vu*^IOgY#;lMoT#1?P<3j4oNktD5{> z@9C6==Y)%hYq+A*OHECNF#XE6?@YU6A7UizTe9la(uWNb_qYDPFx7Z)qfd zAP3=WA(~_RrKmby9+MMza<<@!oQf%!{7h@4oAOvo3tR(wI_mf=p2O#~nQASN$~*iP zahY)M)9=7P6>!=hpb&JHQ;2|{hO~M<>I7zLwd49%Ph8&t@+cd~w~XI)?|~O657-CY zYhA5dz~9@9w3UGs?Ar-}rs>oALj42hv!otFhlT}K* zl!Xs=@IW|MOLiBe1WNXejg?!!z|`(r?Us$|ZN6>D?jk-?t+uzbdwJ7cm)zRgDpF3+ z_*Q8JnVp>y?+jEgzUAl3?Wn7&_C+zNf2*p}q%@=QK21BK1!>1(uJPSbn9rp+yK-jTh=Ueay)0zAMXxp`anhsi62iFI) zmz(6sAA#=Ia@$!2P!WzQo|6@~oh+B9Ts4^gd7-Z7CNX1lvNJ7XLns#ZcEkX}pC8u| z!oSrRKbQX#f(+OJ>f%2wLd_>>`uEP!{{YiBRCnoqOKMh2yaB%of}|xBo)liu D%}CS7 diff --git a/picture/gradient attenuation/2/attenuation.png b/picture/gradient attenuation/2/attenuation.png deleted file mode 100644 index f834bda9c1ff9b488b49452f974f978564504991..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 11242 zcmch7XH-*Bx2=i-(n9D`0t6HbU8NV1(4>h-k=_)f2L*ysR7w&$qVyu5fJm3#F+e~B zM0zI>YAB(1cqe}4-Z#d*@7?i!ydN+|&e-Sdvi6>HuDKWbp^h3Y6+6|rbLVI^)M1a# zojV@|{Ao~<15a#kW$2tc$8$~trt}z*vN}O+_W0?kBC&@Cqw=_kgX$gI409{>r`&=G zL7K2G!PYx21?usGz5|VZ|P=LWV+K4<*Xe%28Xr9m)q=pMc|6Y3eTmJo06>Ep@ z=hM6FZjru1P$=u*D+E(v6ReNUaP@=`WKG7khp%j;gWvZvH$NYz>ptE&Z2tXsa#iMU zc#mzX#QZ0>s`C9Q)W^G~J6|$JUWjd>%NFhkMWL{z_(*v|31xvMT-XMi7#)fauQ{B) zWlSds$(6g8?*qA2(cue2z70e2WffW{hueL=rUi@j$oDA-5fRuTzD!mUw$P1#bF0X{ zrry7{v{DB~_i!lb-JEB3K%Zwe*Bk1axwlt~m#4Sg3g=|Bj|eha#;zt#JqsnzlEZ88 z-%gVqPNkgE?e9-)z8kD`LpM=vbloTiTr;Rsw9w}8QBiHMy2;()>%2!UkI4>0q($`k3Xbg)gI5`K*?pwvdl_Syf+EIe4R?9SfL*agkLuc~&c?Vu~ z7V*Km?1_ol6xzn8{KLO_JQR$A)n;*g@LVNMZBg!5(Un8Vr^=HJY-EI}8|^!nvWud> zUe+di2AM2D_>#ZZXG~h-FIYhiHz0V(=N0|te6)(>dj@UKTP>76eUjy5|@4R_0KLP(mi@BI~x{g&aspBJ!?@Ba4=($+awvhi!E}S3A)}+1-(r0dKIC zCUHeZbEUCNaWXz~@%@SmoF*1-{{qE~{-Fz)F3z4xatrv=|6nw-m8|u(NN1kg4z2TXayMBTi+0;(jOg#>g%XQC9?AOCzOHvEs5`jd zy@pUJhDWi<;UB=3GqMULj4pEhgC_TkziZ2~-*?)uf!qDI5y<~!V7s$rd}5N>_+#dw z*`V#c)7@J**Pyp>S0X>`?y4t9PkU@uo5f+7IRm9KlhsWN zKrMOPaU;v_gI-8I37Y0IwnsOSP8EkK1+*j1wu?Dk$}Xnd5EIL+xud)@&TE2hPF5@B zF8am3?~ct(eCv)=ghH?tVnH=sJwr(gdPI{rLQ$S+?^c=Q<4U-Ck(pKNf)>otflqGv z*V&?V!;_WA1y=&6RyJn1>&Nv>R<5&qrx;C*Du9~ehz@7vf@!r+CRXL6gvRbnRScxs zV7jX|2IqCECYsz9d`3sImyR3{t8e(E&8NeidPa_4*tye7Q>5-I9+lgoQ0-h&UEBOH zSU+6!zr-gtOdtxyWcxn@v;Pl`M|$5~q9*>J78%|i=5VV6pX~*l*)N2mI5D3*&%?Si z1I|u87UX?4?UK`N2l5QO@=@(RC38x4Uy_p83p9B=*a1Y2Jz8(%R*U3pPW7Hd3!B$i zjg&r>dbvpdQif~NcT0PlFa(mE^8G%(JNbHDC0)U9dVM`gBla2&%N=kB-~f=^o1Aoi zlhe}yBTHU>{)-`E%fKxW&=(TZOf2GWnK`lWE+HF;_5w?RKl9<>EXF zhl_kEiUE*-<5})5Lc{5y)5=sMce?K;x-Q^owKD6|r*qgyE=+=$t?29B6~pws#ojEB z97_|Eo66U(UcHo~o9~dtCT>A=MlIf`yfJ7T*!O`S3}O zPO3ZZ$0b3NGRnlnL@Dnz&I-4wNWCmo^H~Aqe(?|rJ=wI^Ytk>*#-FTx3O?9c=ycni zoSgKSn5=)@Rq$?JPF{ZF*X^G6^vjnomm0j7ZhGrwsb9th3>!WYN+0#4)b8@1KY#EU zw>Qk7cc`eTfjPfjTy)q!IOxDG?YvqUs|2x%SbPvlZEsgU@}q~b0PMI*NlSb8%=HfY zM!&wUb<>;k9!MdXkzhsvow9vnxI>3kmAG*0(2!9DbaabQ)c8}`FrsB0D%FC!0Y?cb zC_pqcG$`rl;tD=5FFRgwPnK|bF9Zg2>gVXR>MEYt&DLIX;Y?K;P$szzR`8H@6neF{nzQd(Q~lK zCL)g`F-P%4BhsCLFX42zva!4>f^CPNj(2Buj1?0?t88YLqU;Dhxj94pz}9tDRr_Fn ze;VjA$4o#5w_d#fxx%T8q-&*>9OqH9M1xIi?BX5DU#?G@du=!FjylnOjZ!zVU*DAg zgRA8#{!2JW(Co<@`);qhrC|nOh5~%;k&{5ZiN_L=zBwj2`J3kz7>r9*ODo)9Y@?X@ ze^)yHEjT1qKXz&1nBr{NwH#H{2tl=*S8r>)t04X+5Zfv#!~};yjO5$>HJngrPD`BH z?KH@%U+iNb%QIh2q^`y9e>^8Tyrl%y>mX@mqa4hsz%lwZh_X*njUZQ!L5^gPu&3OV zVDQ~mqC@Km+&VWmzb$UPvrIS`)xIPuXCmf*4-~Yw99}+xEdSTDjJ_p1f~_0;QiI6X ze{}JK=z!S*%yV=s7e=Dg^=Uwt=39bsEeAL4)@Fgv<0si5R9fuA)F(AZo8d2@9_y2&`RiPh3;jV zAnvU1m+^8NdGai>b~TaPi9N?sWmlZpo>e1m9=$&A>IE!z}P#NM$xG8U`JfRV8qTNUi_M`>?+gyOuhc#9?XD_Iw0%2IkwRe@Mr95q4?rb2ByZ@;(91*JW#2eRh+HZW9~r< z7TXzPw6)NqW@pE5SYUE>NW{4K96$!`?d{QAgocI&>t+%Rw)$C`7gIh{|z(u1p8b@V)NT zGm6iPFo{=&*%{G*D3TaiLqdN#Iv-5>6osh6?5IPKWN6-6vR9sMqt0F_j4@8u2&w3l z^`HHR!=`3$*EMhc?ZFl3!OQz4rEk5Q6lp=2LXd$62Z))%UcVBb^&$ZM{*Y!`YlXaG$sEBxAeRIB+H4TaetH`!QLkO{@E#_@h^N|t zd)WoFOBAmIy?VJj18hW(%QT>m2o5>X+Y(HcDDw~GA;_OF>;ZbqV7O*ImxPrg;kb;_DcCm57-T6;0>LaB zM6OoRP)97eHd5Rtr9{p9a4*WOy&+#cA&t-h8ks(EjiyEV>JR@HIpOGpEmysc> zMovO1b+xo2laA_xLf>=H_Nnz9f4DoBXRg zYgMy!YMl}CM_3pqA<&}N?M~|Wt6Vb=VzRu%SdZ%UlgUn1! z?WT^7H(7+uhV4#5nQw`{Tzpvmz%W?qrGdUFGD2GIZ+IL5h|$rBbk++(A~Btv)PTNG z*VT0^ji^%eUNc*KDbv#0nvt1#;px+-9_AVvmlaO;ZRNIqD?5J74=skI!ZDvL#O3lL z%KwgqdEiG@Ss7DCM#lURFE6i0V)>Y$MIBpboKT>}lvhV)D+41VpBpxcyK+|6J>^Kd zAZqh_{DzNzH7QC@RLY4C&!zGltc|~S#R-|xv$3HT7Z<%O1*D~~R99DT4t_Hd8KXAr z)mFaHEBT4BM@k@*PPv>&Ux1pKvDxy2v)z=E8qRYmEiEnf*?P9UK4l468?A8RnE#cS zWw=DQkTKNZzYJ$iUP1xsaPJ796m~nLY z(h#k3;BoOYEH7!x=|tdgu$FdlG3ULUtc7TAeA(a}J-9p+hqLpSy~$=?PitHo2yEtXa2c4wg^+Q9Dj0{`m23MR)o*TiBI2ggt-{5dT&)FS64A%1CGBZF&u}(w-iqRK;>Ju zXGEfTaa)8`bro^VV{uJ}1XGLvOjWlX6z;~QG*8Y@k!a{W8-%N=Ow12Qm}?%S_$!bo zrWa<>wW@oq^Yx9HnWYa%V*pwKN5Hc>n`M1nxVMH03JpgH^+cv{%xY&>T6~rK5?lH;VZ^%u)A_$SP zjwUJpV=2QTi-?@GS3gNj^+WcQdW0?N9x{QScV`nF>KeRMRmuJLYNxF6?J;%p6;yS! zkc2B7p)QIav+ys!01=Df06v(Rl?4ZAzJrL0io#aESok#b{Rl?87eO7XC^_#ANP|C- z&2bd<2lL7M0O!#|Aas_NmMJox4z%X+Oe zkyH^q{TpEW;#yfPgBlwf>z%R;QP#Br-A2}ghAa{iyZ+238dtezIKx*aYFUA^t88q{ zHs770IXWqRwH%!&W65| z_LXaUHj$Kx#tA-oSMD@qv)unxlszK1u&~10r9b3#rt-AQZ`%nSK>n)xNd#lFwUeh; zSA}}^7W;BQ4>UD5@Ugn1wGxiXZ;DtM)W~Ns@?WH0qnov1%cVEuU{=4<DgI~0*Lk(*A(|h16AK>qdb5@@y~!zdQbVW+F~PHU$55-1aM2ba!v-E`Mn-eU1bB$ zM8#yB2ex9`XAPKR>5^Pj&C4YNLTg66N*GJWj-H-gduL~Cr-*qCKPVP0T?Xaj+gSPD zGy14Pb8}<6DLmn!@20Kv&mIdB*LpAli`6O87>>u=`JSEZZNii`jxxy$qI4AgUz%9c^83cIR{!I*&R8;rlh0XC2GrkOP#<7R>|0ae-b^Csd{J7bdItp z4#lDgWPEk#NMCPWneNUWi?)S>xxc&fo*n{qL%oIhV)7JweC3xf8K1jMK`t({I2DUWOFU$gEi*}`~YiGAjI_?5lFtdVH`oS3G9$&9GyK znnf@gtrV(~+`k55H~GW!ba3@rN%42CNG= z$8H4nj+mA!ih!%Hkff~>AZ=Bqkg_9w`**U>oMcaBTrRG04Cw;Mmm65()z~4HD*Q~V z@f;2HCMJmsNfL9?G#E@Ps`ojdZZW!QJspNX_QfYEXZG7o?NZjaN~P`!O&Fe%2E>Xd zJZ<_bZ6Ag(E{He*IZ`!G2r`UeNUBI;(7La1W>@kW^brb&%Or7z4GiQw>~pcrgap@q zVsd5)3F~?P!5J}O)`+U&4&~QbmFbeC#0?O!!oQ|7uzt1};!Y11zSwe!gzBt6i0*I= zTV?CtgkpaIQ*M|YkOQ&~{hKo`|C2M!E@CjWsN3B=X8CPuuLA}EVD|vifFwoNYS!4; zhA$C@;W{uc;A3PFTjr}m(t-7@%%l@63~=1F?1(jLOA!#=*j18*twI+nq>0~yHDZ^j z4M=UcI5C(Al(JF0ljFwe8Uw&bnRJmjoNCDS24?tlqQrHwweXkImF-}bS?o0Poz!blQCc(Ot49Q}fD@w2Sa8)Sf* zB|QoHs;9)pf=4Pk|H34n3E5Bwp+>*_7aF&u?>}eb)pb=>Y?2c%8AIr}LII;L-?-#{ zSE+egR1U?YsVJn9mJrBVfuHTzYg|ZNfc0M+_fW6k?eWRVauhWjE*nIBaq1fxdo%Pbs(Ge^Y_2u){iE;j zm6fXnolvO2m;Y&=ZW!|E6HoY++p4a8&1uV@=lIJVR|IFYEsm?i705Xz(cV#3HKHFBlFY~{PV(!W)${&raI&Yel zkynb=qZi%+!uXMpoXK(!vk=YIF_LTBm!pf_oa@9bEDX&M_yq*`{t0uQ5eBtI8A|@c z$BR22Z0j@HQ`Q>u{%k%=CQ!#uq^v$@ao_dvqet(ir>(wKel0AFYZiOG>pbqaH?hpL zI$z_x`=bWg){#+H6yQe>s{bBlpUD{W*#yzj*7h@!n3)NEY+%4EBt(lwqqQr*dqeq8 z_)K{+ZO5mkEO)(8M}SQFv_e1}Z(s**CThvaZzFJpxP3wyc)w!$;@Ydb)D%(aLp6fX zKM9UFURA9EvucKojScaO&weHFA}0z>${$Py1qEFk$q<$qem443RYrmFmn-5i4-KQ^4L%szLp+$2_@35C)E!g)Vj^sjJk_iyp+ z;kNkBK@pkOn(oRSiu8XmVc=&3+)6yDox)$4|E#VqX?gq_hZt9oqYUU6juM!h2-6&p z%ZP5D5S3FGvlJmSoRnnO9hadz8_^F<{5o=d;c_s{H5jKP0Y>JXx_92I^AET~)!BpV z4mf3f9hou+%u-ky^z&r{LS+T41bRXR2t@*T2JggEfPRT3&P6fAw94GQce4g0qfS~UIqwaNnkGnJ=4-^1V zF{g_=i$fUJt*MdV0KXmTqKE4E`)m-rs2`aO?&K1qs3(lxV)QUJB{g1viJ_W2aPU&J z69GT>HlUkbCbi-jk|9ql_tY8W z>UO{x(DB8u~|f zU=IFLHr3Cb=fF98g2pK&C4m`3BRb$!?jxw@MK0LCiU9^Qv_eGBXJTq_S&CXMp7OG@ zxbkYmPUrnJU=8rV8Zfo^aIXuoMYY5OPLh4xDTRst^q?-m}&>bQK^0+w@^y88wO;hrgjr%_ADudA} zYc*1bG`zbQB2UiCt?0>po~IXT5FDa}2?> zFOmabcS-T10^;X*+K^^^a!(L)+U1FOeZ>3g^(~4V5n|ANXaTS*me7n;XZFjYbsp>L zc!-RU?sYl$4?qx}0}H~{2GF(hD}EgdM*P}PDRC)Ety~!0?jk0at9)fbH{~Hh9$;!7 zpHC~@H z{aD(XQhe_tDK-5>%3Pj;t7(C^je#}5YmgBK$x6hf{A+4!a^b3ckN!_SiG#&n6B($T$mN_jvGTr<59GJSMo{$`n;(*!N9UHhA7@+YtCPLax_>{?r zy;(!H57!Hgi-kdF`z!YSgPXtnscBwXPrNz4@7OzB-iW8=V$sFcfP@b?&K(!g_M|* zEBpG^ar#^hnN(F(wE^yJJx+jY-stG)H{szlBPDADLKtwhn#`>TT%7st3>ZnlXM&tF z0XiL!*kKq9-}~qHcU8 zS+1qA5+@es`u3B0M&-r&ER93!Ey?TSPDLy2LW7Bww+AZ@QsNXlB;;!me0xhBbbhpe zz}MBdc_`n4yOlZLZIRbD%7ui_+uU{3nzC|oBs6vmvJo1;{!A{hPJEs^th#a>o;^1} zOTd?o9zKmAw9c|uI++#CH;RJlyE!Ey-g;ck-QG0)dcg7edul~J>UdMUJ}&-7f{K*n z>DmB&zXeNY9j*5I(AnB~-9WlY?KpHgq-%|UudoQINbgg&mBjCbBVL{?J!4gzT^2pZH7rE}Q-KFQ z?3G_Fg%tu5OHP~*j{sVJTrj7E*E;r@uzzn(#`E#zJa4M&!ARbl?OlKqjY zwdJP;6sbN>Vn2P0mBk#UR(029xsD%5O!bCM$#eK8dAAEW&$?Wp3OS5>Um(P@ECt;9 z(p_sfc(o{BCLGvaC{_7Cy+kC{R>h~KeYWpOTbyca)Q)2AtF9f(uk-CG?3tJ>YTC>H zm7$EpbG}=JFA=L))}%yOroIziRPNm(%FB|2Ou_#u*7`Lo%~%lTZrIz~YZCuSronz_ z!FAne`gMikt`DiAOU_y4KZ?23ZvXTCYR0?=GNBMs-^5? zlAj9$c6MthkjUWOfu=u(OKinF@ahL?>3+LU3N#-@^g6TvSNCu%m$!p?=*^aD7e3YS zrd+)&e6+cmbvp9W2KUG(r9k3{eLAV=0PsSGtKfXVwRN`VbGjYC==`J^FPtFjExfxr zmg6Z+XJlx2&CRwgycw{h?65OPZvVwU0rnd`$3KFpH`mR~%_D#-bBmB?z?Jdf7wPe- zvr})MZp_Gc`;w=fECro#DKKQ_BsBIj7g`8JEiElw+6o4&D-jV9{;OAc<>k!@rJSyR zK5Hya*4GX`&AoR0x;i=X_rG-ZK-Enw!w?f|GcSrrFX>&?jrdP?v`PGo2OET?h3>Db zU|FVM=9e?G*7BGJs9$KVg3p!-x8h)(CY(R%k=NbocMK~LJz=aOZKY2;KMmFP6zcfi zs9LQUvkQ~Cdw0OPXjUOTm)S(--2&ZI_xJ!%m>}kN_)^fafz#jL-z0w49Rh)nd^db0 zy!WTqj`D{Y`j3>>edq8T)!nZ;!~MkFy*Pw9 zCJTj;fJ!~(P zW{coAfQ&ZxfT_-G^(bP2T7o-;KT7~B4%&DwR5^BLYV)6N_H*+%Wwa95hF*PkJ>CEC zD@IjMTuzXUhCKftGTj3!Zres9o!>jqm2{VS78`|qOGaTwAONrT*YG15rqqWD&2KfL zLlw@!JKNiT1A}JB>TDt6b91jBT-Dh2sp|Y~;Qa`=sZ;xd<{hzd*zNzSFZT2&6S9s4 zxy%5Z0q4Ib;lHhI^tf;7x5JG5^Ird7bwim>Jl)1D0=8&IT5caaSo+{oLj5jUAT9d# z-lRvT(B_J((yv=rv+uDAETf7*+N#j~=o*lglEHd}-@5-j}>RD@YyxJ|t1k>AvE z7mJ|M?SR_(k5syaix3q+js(M~%qD8^;UCBtmt~Baa=FdK16#_S7 zvDo9|OclQDFfgoB^;a){b{dZd;5R@y&=_@KY!`jwED*ThYkOl!gN+5sdO#h`{FKVK z?Ck8ewg>2yC}aW3ng8%M$@%1z_0pPD^jv15<5mbC9W|*o8!omEbQan_I?4i|XMcCh zZO%hXEXyGnML?kjgR5u?-48aOJ~lLT+zLT40m{E!btY*+q+vg^&M}#iMpswYXCoCT zN}2_VnLe=cGGy=`sswpY`cRM@EgX-qXWgqE$7-{S=tbv%`9ypCi@LqZ*Cq`@&~NUB z=1^J2tb!Kx8&IP*j3H3G%4SL`Vg2X-|JOpXGj1}Ge2%F73td1B?>P+>9a!Ogt9Snk D+VktB diff --git a/picture/gradient attenuation/2/no attenuation.png b/picture/gradient attenuation/2/no attenuation.png deleted file mode 100644 index ad4f9ed4b55ff740c026885a78699209fb3ac182..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12464 zcmcJ0WmuF^v@Qmr3^5=jC4+=AGzdru45*|a;LzRO(#jA+Nq0#|NlT}6gLHRE!_a-d z@t)`0=iYnnkNe~JJpORc{?=Z5@3r5x*1ML0ALQTSJ%0Ka0|Nu^{X3Wv2FATu;IBFk z7BC__?5>7^@eJcVOicN6^6uOtc#P3J)O90!JsEpNz z01>CY#Kgm&#$`igvU0uW9Xu|-5{5wqqm_28b5F=VDF4QTk%}2c>dv*smG&GE^t*b*Ik?%fDO--#eY2;7hLm*sYAW{fK=?TIQiG0$o zO9z2Sa>9W*ivK;s6v`#fSv_z_Nl8oMpG?@Y6Y=Hk@b1yzU8uug#($feBg+X1%tSm+ zZl-2fa49nMBRG*ktgN{0ZJ580ot+5cUJEM_ zkq&&zTQY+Z22KjzD-_VrBqUZ_dvG@p7fHsk{q-K31U(V5;Yu@c<(HU~2P=dNN69ig6u`@4i;x3$ah+5YHDk5D}-ZFOTo?~r7{j`9uoC@PZagwFpmdb zrs?Z-R#c3_y15zG7v_l*xlZ#EN4pp4WqL`{gA6)#dZW|pThm(o;Ss85c?WK1Cp3|b zt#OHWIEoT19-$LLt$Z2WrQ0$dIlHvqChU^)XBvzjm5yLycecI23GQJ2{ot3{tE2vj=jRo-s%i^v??j*A#4;;C)jCY` zPf_s~%P-mrEbx-AcBY=OxY5h`Rc*71J>jYMhbcoH3QY?*O{6h9Zb&KGXmUmRSrDRMI$-aoD#w>j6TmCtr_WS7 z2`-&+ww&%hWLpxv&hTu{DLAtLM~DdX!>?y7ZpkJ*3Oq8m-#;*-rAaLQu|{0x62J|L zz2oT`iVwmiY|F3enpWj71WZz8;Jwsjs?A>TC> zj9Ftu(3#EER$S}uPZXyO43;`>Dq(8C3_T_d=<*TE%lZ{bZwwE}p)ePryX&KB+ok)d zZd)&StY^vho9;vgQUtMaadDv|ZPzt0 z5L-p^qMk=DNPEb=z3k(K-W%*RpHKSSpzb7gx$}b?0E|>HQ{>wNt(wi|u9XPo? zc!hZkD9onmhLq2G_P0fGgR0jId(*WKF_Zk_)qYc495&ul-?lcGW`i3^dU`s!ptEJ@ zp!)^NhyusV%#7=MMrNkn&E<*u4E=p^D$$!`;(=6QmYHKzFMIDyUmFRGn) z6REr`MsvX^6pF)XOS!GRosN%>tj>8?JdRCk>mrUWpAriN%8LK>>mC9dy%yg` zWp8y&Oqgs0?N{Hq+GM<>q@=9BSWiNk7#e2n;HA8F${(M2A{K7Kad&Ck!@HUHL%O}m~spm~p|Dfk0e zLR){*`uCT6YQ0$Sxx&|%~W=42gL_~rDbGDtb%bujy4A4bV!dxA>R{uEg0RQHVr4FL_|cZwiB_e z8YT8?(ypRle$m@=dN!?EUwrDj$??pdvLH* z=k9U>%d9t^)8Js}EiIjZfZC~T+qTeAA2*j~TYWEJh-~f(857Taq(Fgz^L>Evt!W0E zW|E5&PY`bs12TR1=08gmwGputikvSZ%=hV4O#=1uNLkgNF*D-`yB(W`S0Y`hI9C}| zQv%1dPESwI&sW0Mtn|fxBf<2Bxeu|hGFN=njnJm_Xh<9T<5 zG}n)ah){gl=nB~9X8!Ur$?1O`TP1intz^q3#isq1g25_?H;l&|TZ&)FiojvHI}1%Z zyT*iIiqM0D12^GR2|XArix~?S41N*REDnPO>6Z|KBQk)a|38P7B`hb84^LrKGC#e3 zszD(66o?;Br}%DAi!0fe6Aa+sm@9nbHsYsH98Gay#IvJXoeZ9!6?ON;9Z?ygvF<$#hJ|R(pnof90rlp*FCzDJ=|{{}ge~XW zT}CjN;vtrdFj9b2@$yG0qoFpOCJc81^-sesa_GruO9fa}ZpS|(a>mH1-Cpfe2(ph7 zA$jjMRo))HV?MxIe|{md`lO_Y9Gt-3fZaP@r~xJ<#PIg^?n@CoYn)L?6$(rhc5fLN zz^C;%5W6@5(B$(@X(u5$ITCorP|fNOY5hei6R~n7#}^#Qjgh^z`Y^-sZ%3?9rk7gs9rnb%+zCu7l6n!B6gKxw*`)I7>@QPByvsZ?9qQnj^lw zIihJ1k}NuIZ3;2Ygy792ZG`0UhqmX}T6uYS2dAgwmxq#)l3dCi->8H2TK#!p#3c0h zKSobCrRpH37E<02V{3p2juh+EU@$!SVh5=H-g8b)aCdh%GZfl(d9uxE*hN@K!{R0g z_>E`>7H;D{)5X}Atv$oy)!N@4N;R#2A~2CN`>pKRvfkdP217Z;a`nIYXas@~h1t1)Z;+2@w4!*gAKuCMLw z6&y{qZ>|A&n0CCF?Hd?iyGt>d-wt#)5G+WZ#0#}71FxgEKSUMu3w-w02VE3R2(;O$EhY+{wDvH5>GM*pTa3t3z z2j|9MqwieyF+!3d4D<6cA|j>qYKK==$D5p?O6}4QDf<8+DJfA``8!DgOK7GCy*|C^U;TfwI=?R>(LGrZ&dM`G9_nCoQ_17~EIbT^TXK-qMjX)WIcm z5H3{^>4RrmLH#t-k0y?S4#HR;0cg)b0gm8YrV9Q_=+sUcvgecPwHIG=h~xFzNdI6g412_Kx^x_)iAFPJ zCSa-6B||2erb#u8o?-U93G``&U?bY!T+|8xYVqO97!LJVpm!qW)2#m+X6C8*3Cg<7 z*ik$gKv@-p#s|ehawryCeFdcm1!d3*nFEK}nAhw}V=YbymnDdl{@}ANcfF|4L&2z` zr9n#cDS{{!!g+{VTRm)rn+Sh33Euk@VDQ<+^sTEt12`gGuY^1zj^awub#1|=WV z>(!G^wsl?9)&pi}VLLDG{%`QhR-@0&ORJdyj-Q zegaO9>j8+An9;wf%|k}ji*A0_12A`Yt~QNbV{k;^%4$O(u~U1Sbm@Vr+$y7969amA zUMN;D1z>i!C*lvM-FP~(RIqLN8Ds$4ig;vD68|Vp>J?ldopP3PriYk0s%Rc<+J=D3 zHf|fKDJY7M`GMKNIuk52g*Kv|5%&_yxmuEY_wF5>opsLEIPBNmtaeBJo%+SJv$J#1 zj7>YbS*lz&E=OIe@4>f9imcT?tR4W+JByEVsXfbvm{W zjY4I$J;MRpddKO87h7J=bAEXlt8;pI7%}nUW)c8(-++K#df}8_3k$Ai)dK6FZC@RQ z9E0@^`s#kPsu+N(W}7#(n$zPY`Mn0285t$scV4caKYt!Aj4)nLK*oNy(#3d1TZ8nr ze4l%|FAGy9a}z$&KIT$VQtDyP$(Wv8Y-adl6mAuF(ngpK0l?|6iB2SA<*m)lM z8ui_FOOZ8EKtTVTi7B(Bgm8PJxHFti;^}qiv2Mo3Syk5?5RNq?3uIp~E)Q)H$W2~& zL1AI({{(A{uV25C^O`^LK_JkHk1I|byH0FF)Wy+;!RgKv>MS`q`6;bTPf2ikt9=VWj|T=J#1Yv zZm=J?Qq+R$y08k-&aRJ3>K-!X{X~iFj9EpJ1WXFqEPh5ovJ@PlCju|0eC2 z9bbHADe6j+c55%SyG|H!Pp>3=pyR~Sk4d8bV2L27WLwsMx}Og6pjnqLB`}3)1D`-_ z8#s#EJ;I{Ze_<{fwesUSoX6q7$)T)c7QM%5#O=DR&<^0GLCJ@P3Y&Fbf!5b_K%}&O zV=Ikb1-&{lT}wmP@+|gzC+^qz(zy|OYcKUn!u4;@x}Q0kFZq0WMbms?@h{3&ZqcP{ zpPYd&MU|$P{JCb6PlYlsuj|6rfjbUDIXV2kR@4f&eSz(LcUq0smV^WT&hEx1QBOLp z5>A%pQ;eUL58T)=cM8I2sl~6WyE|fSgh@Ub`RC7v>w{JT>B`WMNfw3jw*;bmeV6Ml z$@R&ZfUUd>bHtAw6uqMw7#QG^9Ud7e^|-J{=YQz=@f9%)q@kOSncOJ_79ur1WC5?> zodWPd9i?!4M~D9P`QhK$8g4%C3~8a1ol-b48yVNpVVpPZS`z_;JKdqg*UvBA`|g$t zjs+tBGCQjYSGVu+f4RN4@$D1#!)K>wz3|4PWb|%?OU6&W_QtZdF0}@rAqMzkXl&fG z#F3GoL37x8>o9RRI7$3>XFI4wRc;jUn7q$Y>d5O{_NCsu!2r_xQnwTHzEojKHa0du zaarF6?TACqPpW?bFlhv2qLLX2d}L#Yn(8D6Sm!jaBpDjIQ~dmGc4Z z{yXJ50odpp$5aD5d;1o2=50Rq^7?efhWW({F-34(^e`gy+d+GyLjt^7SaDB$s;(gb zjRAg}vj$TodJljHzIp-ePvU#4uKrZp^F)PVMvC#>vAqewF7h;0KECmmkj;2b8Sn^M z94QD%-|op$gNT%rP=GY~JwKo70L>!7*0AtEX)-?!wxM_16}yN+$_5NHqt@bdGBm2J zC#&r;uN&OY_RhC{7DroBCX`gtgdW}^2lyT7a2=Nay>q4T0SD-Q;}zTK?hG~%$-AmW zJ8e!B(@KSqr^8pe%JJ|ITW*gE*8-!K3TIc10RHOr8QglIQH$oKPhDMI{kL4UdGaU7 zB37-#=}&KM8E^hQD9pc9&z z%VpvPco0|jckgiUX@uZjGjC}2JN?&`sd8p~=X|us!~(=o*=Ie$#L^@4jfz7P1a6ZIquyA6?}~KlhB>WmAyef)f36m4%ec>l|B==v z;O#VTyGByj3S7Ms{g|l71mLhK(ozS*v6a0O0;M^^fWHZ*4r#y^d*@~C?YRK<23cP( zq;_5QS|2VCN(gLiMVL5K;ZAbJTE^@AiB-=OoxbU(!DQ^Qz0vLYMURLXyVhD|;@D!U zfDBoUO^vipDd@eIV(i`Cy54f1mmSK;)HK&4wMeV3hmM^~ZiUu7^k##bbN}k&-{tKf z8m~RHA(b2cBVy#Oc~tc8>-1HGA9k}cw}iI~+O4Q^HYQzi000L>>Je&x)*L>LhSmos zjYc)wmsK_A4EptaNdSCBkPLk4YZo~Oyz9G@>0XFSfOAXguY~x!m4B`pAlSm2DQVUb zoO2$B-Ge42H*F+ncTfS0D}LV2(|zbJxH3qIPtU-u1GtZ&2Y?*r|8NI{CKbpiCfp!t zG>DCTX%?5tjH#m+p_-!T9JW6gi1nSOeFMu!3)-ZerHZ0CX-#Y(J2**KMlUc1j1Rdw zy}_uhaBkJPa{%QOuc94Az7HU$bGnBpIBoYaKxY8q+MCoj?_vH3V7z-|iYB&t!$$9oy)CXOmmP04PwS&fc(L+W>wOkebtI+s9ia=8mJ-!G5~Q#y0%Ud z8xay;9r+xCZiV*%`_xsy3GXrt*_|hnsQYYiTRC!WqmKp`E)5VV7X?rz{Cq}`vbaA) z{&9k~<;PAEaF<x!!K-o^;!ZL@&SsETHC1NrfC}Ec)!UqWIioH4iwqUqF-)Wn^OI z$1v#Z7ZZb-T_Ex$$#0e_bIO|fn*9O${hHD-+;(G5gEQ{eWO8=*U(G{(@?b=?`FqVFZ~yzR(nw5U}}8-zlq zzu3;B)8U?*zo&y%=i3ua2K!SMv&?Q@e&K3u568QR}nE0c=rtC*ZdUudWz8GB>;Xu5P1+AnJTWzOuNWy;V%Uw;B~m zGKM8kE-o(Vxw*M9>$iwU1O%P`I1Vg|(&#=9OA&j;Q$wQS`7aeeFlnsz>D)OL9`Yg( ztW9YVWHWeAqD2^)S&BvtVBtfysi+cI5XdJ5)ZSa={-RVm1XA ztFHk!Nj>Sj5C_1Dh9lwGR6DznuqM#Xmve`NNk@LX1!`7GE(e$L_C-Q=foRg{VU_u` zE~?F&?Xmxp#*<74&HI?sGb14`ZhNrUjG8GcE91~_!*Trk=N{dFu^?ZSmF6|z`B`*H`i`4PHAz@K1 z2tWH(Q4xb3L1zcRCz6BKRkNSGSor6xbQy9;bg`VmTfA1S3MH`x(p$ z-KZY}bqhpsvCW0ikOg?c%pnb6#adchAFOnSy`3o1Ivkc^r(+MhijP`CIhJ+$2F&P= z^b)M(BzSV{9y(V?I^Y)HZMeYd>+lp=&U!Brd?3~dve}Zw%}QDcsC?pN-npFWYWP4E zGQCtH0Kc0?*2#fKTvj$jla1-y@v)=PQw3!LtHjqm{TOvq2L%DGV>U=DL zNlJE05H5)uGPgCY5Ym2fy|%J_=A2f;7|>HmQx^h6^DV92_zM1?yO+ucX~~M9^qE-` zG{Nx&WKk%9YP3gr0+l9vR3pUBb*(3+Y@^I%kfyr2`gd67&2XYo#TpJ9`ZzT^RLQuNEQo|T3A>> zwd=_rV1dF}+8AZyzo5xwty<-I5m%HUOiBs@u#WO5L4JPS3)I?L*d0*s;B?-BQ%LLI zd(%FH3`uARC4mUKcGC+s;n;Pub+^O^J3rT7I)zQSgEu;b#-?Nj<_IacVjbzYbmPiw zdSZ+gsN`D@g9ZLL+W+}!^-)dUShsWCp5$r7@Rr7~U$eO3^T?w09_ z@N{)=4+hL!uY{0-qErc5Zba2CvxyzEKV{eGH)wMwRM*A?IPeAo zU@oVkT#dq)93d2LyOhxtTN=(+(6b3HqBchs6KfkS?tIH7uU+lVs8>a`;{XHQv!lu5tR z&IPueji1;G_L8Gs8yrvEZW}h?vPrII%D_gMEq@U`IgIP- z|KT!JaL+yyX<~)xa4ytw@7Q{phgHynXF|i_Otuzq{)X$xyHO#Im<;Y$MFN8dNmBL@ngDkTh}@E&+nV$?i)lyVLJti|LL4&zSMF=JjNTg^V3f zo?}DqnhR_32mijQkh0WoAX>RWlffz3*@6<1FNN7m^r~5b;fF`exRjh#^gcNWQNn{Y zoakPod~Q4I5V#uSw}Wx(7;EBm2i`N%f$TWH{<$UI!>i)JO@Qgs^3rjq>~)v0klXLy zL@plw5#k$c=9CA%R*J?!7!RDH{!Ue*f<|VlWb8PvvKcM+fN3kEYns>bwwUcCr*KOQq$g^n1Sq1<6>`vu+!!?Mt27qSiQOhDI%G z!oM%mn6iah?j75Ttv^1;0r8{$@z7ZXP;h3do- z7FPpWo!Rtcigqh%E^;>6Tlamwtjm`@zK1p%Bq%j^b#^?nR4D%?^UH{_)Sm;J$ime2 zqGT6L9*393>5Qr+imiarMko@;+@H1(m!MJ!yfo7n+4y}StLCXYf0nb_jCzF^3@E@T zGpc}#a*pi;c*5wE<91NaN`6(-EM`)j{}bEvP4Qg_;^_! z695maym*&5Yrhy1`d0U>f`NwMI6CHgdxu#;LN?%z!3hkU>38Nw`0-PsZ>*Cm)rdJX zzG+8O<_Jt_7<>R8YIQ+Cnjj1IC>YpIpZbr`5iD%|i3E!>GWz@Ly*GJfM8_ZL<`csY zrI2B`3e%(afoXs7OII(3PH9k|t8(eRUc0LJ_iYJp99|3#1*ZibaXo*1FQUz&t=zQj(sM#nn25oXtHeC4kYf z&xD=_I3`8mz!?4Jc{bE}?{EA`7q*Z1F@eho`lzNr1Rfd=m&yP}Sk{Z+DKIy}2WIMc z<`CD3u1Y2vK{Rh2IPfe^wZBvaH&DGKcIx%1PnNU4-~x^+_bYNaAq@LfR6wo<<7qOu z)u{J2DP+(~m?aJ>>d9nisA574JUqzo1I`F`|CTm8;bOwl&L016k?X^9q1yA>o(pnC z@{y5l?cMcp+)sXBaNb}2aXB?pArL*z_xt0iAx(DCGuRjvqZ`dR0uw2U5){{lA4<`EAF6DdCMK63t+9;_0 zYEzVNFH5Ml!mf2IaBl5Bh9~g$!4v|e6jQ+j6%n0&78mEL#9S{DF7kbYegkLAMaMcv zO?VUElyCLNJsF@c{x|Zyp}hRb#~k^XDu7X61iBq!tl0tP2OW~EMk#Fl>;{TN*lu{F z;KxHlL&)G@Anv;l#Dq-Vs?V)uWM!k@%F2R(SWId2)DM>o!odL)aTy^Qum4z-v~_m& zrXthQXf3U*IDt+B%jxpGVbA{=S~*Pty$5epRY`$5;By-pIk|S11g8vyWYa{b(Tz7yR@zz1p%E~{qqd0D)S^YfX8@)9mlxi4{a!_7ki2kH zM5EI3YfcVROG}HnX9`e3RojrFqM@Ow<%>rmw`7Z(A#Sul?RhP({ZDLFRh8=5RE-1E zE1UTcU|YR(<m?cskaj&w+*-&$Z-=DJCHGgJ>!)6rjT^OK zdWl5K+!2F5bJ5t^Tf3KBopLSbsd8yG4LV=<)tW#qIpjb)f;8I7l%mIgUZB&;_~$uW zW)AZqdH+j{zZ1YLp%&!4P^n1dQ|2T8%5T(y`EySi z2#pXl?mL_{m7t#PAj-VLY0uNB8xiH%I#8aV+1%ABuI?B;U2*mSnKbhc~}Ut3dZfKi@+9vA2mL;|q3nggXABN|7M1@^!Ll{8vhN=6=f2aw6(X76pqW+;E#W*$e`&P z^vp++O66y85@c^B%NrE?vV4l$UpAOo$L{YRI>-Hat=()M{vmp@R(k!(qd=xqj^ML` zT@m?gn1|lqg{G#Rz~Gy_<9?6ZVfA5PIWFq|y@daBE*ZI_ZIK?2xzNY5{_32;oNSU7+fUF;w9zqM<| zE+j^w`_o9g_0?+N+9ID1bU{@7ABwFO?G1p=96mNa02HuV0WkWn`>(k^h~8au$32Kr z{Lgluljj4gq)ZusQBWqi#OJueK`&d1>rp`k->-p)z4zBVrecmvbRHSgiy$S+iDI3& z2_^=Hbe$$IRMxA*c%Uz&Yg>}_7nokEi+fdaGhxeo3QPg`E=6H;Gsi#t=lP)z%~qgA~w26F|Fy>pka_{qu1uFx1NYHGXdbGw@6Xq+dU1oR$mf!V%XMpmt@oUBF$i>BhNL<(Fe*n*V7xDlA diff --git a/picture/gradient attenuation/3/attenuation.png b/picture/gradient attenuation/3/attenuation.png deleted file mode 100644 index b8b21fc30b8d9bcdf450bc35ee139a82cb739a6e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12469 zcmc(`byQrz*DXj42@s@%dm|koxC9T-xH|;*(2!ukEx1buf=dVlm*ByJL$HL8z;*Y*mSKV8;&ffd%a|l;emchSIc^?A<17A)Su8x6mD<1qR zV`G9Vl1;mD7#Q>za&U1?_w=n99Cyv5X8v9KW5P-dJ^Hj4*msGs8Q#HPkXqu)-WxTm zn7A$dsQH_1@%Yq6=~i*+Lt9+?pJs1J?vYe$X~$!Sh2h`J-Mq`-k0FQ6#OhCot@r2b zO45wDQqhUa+3L(Ub49psAU)+Eiy?jIa(q!0{OD%M z&_Qw*I6Rl4BPcA4$2^&k2x)}}fkH>3OeNs(R8GaPFj?*p93mo{?*I8`KZ*qCs|iz zFOGmH5suskE%D?QJl*N4C4aMl4RVi?zt;G$V|V(({cYcnt*!7Qg7HJqefQxmpH@HN z$G5`wI#gFw-CBPLM<QtZ0!)CZ^5Wy~KXaXMvLN5~sJ*8|al)g;GZl;UU+O4I)n9K)6s)!3(O(Yv zZg^2;({Le9z500k+QLqj=W^l-N0k5g4#%=fl(2T?d`xzKrc85$ z>`dLvv?xWs1D>Q>ofJ29d?tstkAUm?cIyVb$uQ-|6n<1qYUBl@JUm;i4%HqWnSF0ni zfYK_(9Y{S@CX-BJXwLg-*s;O$n4+A0_O-i8jmmk5eXm?dAal#PkC*Qi#9lILa)mAD zJo4|~*#n3$^37=zjN-`Z<&EIrW;}W8t=O@DFLdEOmTJr*>o!$|K5w?-UeR( z@L?*1T4Tl6{HA=*jP6%$qTT10dd~~u8@sLO%#C`&eAYu=KPPS0WVcJ3>2VOMDRP~x zI=5f6nPVJmC(yhw4F)o{$2lk)Nd~#KCwIL?!Q2N((9xXhCYH6I3$$nnR~i{g zbT!gQdgoVMQEO6=mzV{#(qixdev6QI)zA6l!3Q@@Lg`gwU(|UAs(+5#*_~QLB(?QL zG^_Wr^Rz5Wu3CyjL$mu+DZ`3iXB}EfqE<%c4p1uww&brNlC^1<-q9X3IVXj*l~cVJ zTvk~PbngU}7qjR>F(GcFue zxr3gfqu)p0XQnr*t+LK9_&m9qz22A4QudMr?@voT zw+qyAcI#TNN~0Cv7B7c9$=>yjyxka6${6U0qWHBwu3PV9NtGDZyI)}?F)CqcY3WJk z-@Cw}7=}38i}i~-SQ%bK%Fz1f=fe`&3?r(nhuZ-(?=&4e*lyer6+ZkSiTKk)=SOYW z@YX*zmOwg++<&nrcD!jw#djalbU320ev%>N24!NpYu~hodHeQlLUQsjtzvBot;~ps z`%UNT#jC$-iaobW^r{@L&Rvdl9UVC{J=Y(fo}Yh`41GW@?9Rbv*pN%>wbOh;NJ@&y z&d&aJXO@P}_taG1R}p?K!&Ay4DC?;cF!-)qNZ9~C}68aH0-42PU9M6!== zg+pLa5fK^>509|u=&pejZYrOdNHX>!!rs0w%Z1cuBDUvvX6R?Q98!+U__Ck z*v;86%6zBwrp0rxq=eO=-bsqLw8nP)fs~Y#?pFtf_wn&+y1HRAjc$xAEYgcG$x)sV z93;~BAnngbpNr%5pl4RYnWZgfjrH|~JrC$a4K!>EhP%4%wO${UqlQmU-L*CZJoYWB zdJ-tRoQFm75fuCB<3?Wi8)Ky!O0V}{5<@8{C>)%e+Q0eVncvzf&0JYs4KweFnLpl| zDq?v^LV|LoyQ_8wu&wlceEib0=B6fU_Lftq=izF$^4y3GJDYLK^qlkO$KNNZhVxj<#ZRB^EpnQ7g;uDh}Lw70Q%RGiw^&)ynUUCo0^sIlxPEir0-xdmgWdOgMWcX|1f+s>@}a*FluY6JKYPz38p zNX?-NkJLFd*Fyn8UH4|Dz1;WcX?(&`A1tdg>*mth!pf>BHHvyARO~YQ{=+pH0nfvN zB4)WQ*>hR8GG~2OZvxnoqswu>t0YQsX2#s>(;hLtudz`@?^*Ka z=1kzy(oiW#cygv>NZ25fbS|iw9!Xyz=97Ci-8~Nx4K_pO;Sm7+QZ3>Nu4H z$aEZGI)ZI2gT>_Qj9KiOg=%~Rhm*#lufb$4Kl%~6W)5_qJ`p!ZO2RFElCvl!lTrnY z?91B;IEn9k0yn@DA_sv6HvMdjBoh^i%W+c+3uC41$ffvY7U=k>7$g6{e%gHCA>AY`IVezL=>N^WyI;N0tqP(6E`usV*77;J4 ze@mdRbxXN~gcd`D7Q(|Q+c-D_LW{Xsj`86mKMi=fGsD8xN0=4W)zyKP^xxfeVAZSq zo7wa9>C*~_d5I(Uw{LkKP(N2aNHz$lGj8<_3lF#1vt+J`Q!@;9`~KzG*89mAIk?5M z0a})&r6t`jCbvLjA`=rsK&%#J)%2OKt*s#x6csx!&ki<(uhH@SBU{p{xy;KTnfLW6<|E}~{+Zd*-%btHf8@Uf^S zIiAPmgN~-zfhk+L1Y9$%s-ED{6~mF-3-U%41ma0RKtNnlQfP&F4>~-^In6pnE);%f zhAR2``szQVX%_U#w*SH}DwG^O+0j_nXG=^pdAn*{F*gzU;e(d5kdTmmaA3uS<_#V; z_P5bO4Xr9YNg%{Dvm)%SudiQgU)$V~a9~}*R5pBKS2fNAt#q{_h;IwTDvmiC7*I?| zNHCdca7pH}l+W<_d%TV+*21?O&J@|*+Y>i2p^u1&P}S9SAx`=OVl$$}=OVS{Qo}4j z^9E-nosh_9cyrKTXlO{c!t7pmEM2*@>tLn`m71Cwpq9^`-m^^}VX?6?>gw;;3r==- zzZGc}kD4d38LHP1scCD6&|R&{By*Ss==JZctc4((FDv#`A7s+eWd{U@_09@BeE3lF zd-&w|xXw;xa@2r~O}}a9$gLYXTY`m-ddblSLpZ!P@AHb%&^!C!3S`Fo((*ta8}w5n zIZNr>_CnC9ufMElVx&GZptaC2SvS0R?e_w(I{EJvN6(XV7@%g!)gyCnG6<)$X+%Zh zO3}yvZ4ZYHvy2RumXO@w*C+-4h$FTKM94C77D`1LLcIFZPk0a(Xr(nC1i$@Kgb4E6 z9~__FZB9j=_jzSIY^2bwcKqf@33x75NVs%-PT<_a-*-n?lSC@;<4Ds_YWc@yT_nxr+>H5#$j_O#|!IQeg>}SVmbs`seu>IH@CbQ z)c4c>bm3HLx~XGHE3EgLL-BW5)cO;)(ii(`KIK{}@La4AIjZpHa>j;pPm?S0Apr*H zMF)osqf@kJ3einDw~j;hTCA?nk}krE=1Dmr>5jVZF*6ii+P*lSHv_ zEHnMz%m%H7MQ)B^VTv)PW>Bu8;@qhIVnU>kNpiC6+c1Lm%c+C+EpQ7mauyb7f3YI} zsPb_}b3{pT-68tj$Z;wvu3~YDufxdvn4tPS9Guy4F|=X08@cU4=xLsrCnqPfUMG9h zV`Ej_KV-X4gzN*$+nKepIXJs^db)N(kof2^NsYJwsM?kp$wpO8p-}x!4$eFEx%O?P z>fSC~DrhiEx!ci+M~in7n3Yc@x+vccx5#;m6Q7Xq5Dtga3ky>#B(m18I8vNd|0i;- z?CtM6;gNG-#KTnhbdbD`u1=aAL2lkj)&Q`1?TJCGzH)p_~XZq)F}oAhILW% z6TK#PPN4km%k<<$vhk(@zAAhsW_+^|A6OgG12Isp?%u2d zjDPp;U6Z+Q{)4nI28LS~Cp$W6t*w3=xYHrD(KsPZT4)HY?cOwn)oV-IvM|T$!bD?J3NTbc^cvJ7FDU{X|o6dXJ zzy41E3)c$M#`^0`F*haji>;xsuA=)fN%2&>r{QBpKPT^sg+rY(8es~Qo}&>f@n7!n53=62EBp!n z?7qDyIpMlM+;X+U>EPx#()b(>KY6w05zjs6LZ8!cKeB&!3FqIY_?vzT%oe%)F*&8= z(g^_|9}@l`otkEpV?G0R;JCC*gQjv|ufSoy%p_H{)xrlu*;mmN4SW~D@YRLj-WT@X z-8fNj3m^>z0och@mNPPztD+y$rk&&G&xB<9=Ykknd+2~#0z>ZaI`xDZT~9c0V3Cj* z7zKxwkafVVoigHa$0I9X2NCG-Mev4%4|Pkq`sb%{k5v^$hLw(SUdg-l>EBmLLc;Hnk2FI9gsK+{*r1g!!H~H}{l$3L?fh$J zy-!)k(3|nBo1=nI11so-G?u+!CmP(&ITTsp?(bmdp{HNaD-w5f2K1)KS5H%&ZZG_U z*ik5$YG3w-{<)B#!w?RU9t~J(@91qu0k)sr^MpVr^oD8&nVOke4E24ca{F>|Xo-eq z3!aehV2=VfjHc-fG%#xigCWy~=^O%r*pGYIKCLRJ{Da8*9i6xPq^jN|B^JM6fL3CH z)p!bQWrAE0p+`-*>_mDvU_B=X2UYF`4HaYQ?W3uGM?epZ<9+>tEbv$C>bL--;YI*j%Bz;tQUG}t;Y88mQ5_Ha~~&CJ{kZtWcQy3xCdz3f<552+TS zbbm-n+WxcREJ;yLP8<%u+uqSpVbS;Ai#F-S%dZaEp2C8Hlvod_RZqys$QG|Jc672y zNJxT{jD7VK_DZD{UtzsHv5i$Bu-E`6`V*#rt;fF5PE#OA1Q`Ibfx!Ffc~pd0N6344 z2#~VszI#1Y%K%c#KrDjpgvT0$6cl<@Do##ZBfoy3YTb5>2C~GAuiiR4cf`;L&8@A4 zkBq3}(TP$+V9Ia)6h62ObywHm!GuDA>|B8P0mL~1mt2$q}k@m5Xz9I*1~ zTp+>jdU1JsPfvc|%H~v^3hFu2I|D;QblwC~p^70+H1;%cg@@1~~>T=WCr zvdO(HCzF9P_@+?4FN~RCtZ{i9NXo!bS&#}weO5^NWIJB=c_&7N^ zv4Mu0s4(ZW=#9gSjZaPvL!+(|CYEmF4`te~tzIzRqR1%>cYLuws_}}aq{PQ6To!@1}`op9%7GMRaDX9&M-FU|jBOkb}{Rc+p3(ncbPZjHw zYx5?EcpVq`CV%_(0%gE_1zP}F?L>=@kmt#E>C!V@5_>nYsOZforBL>d|D^?%emrC$ zrsuXE>O9&QuK*15wXu~6MW~vFZZ#ly(Z@m_z_1ffwJghm&g&3VH+lkwF+*|Oy`Me%=v!QVZ ztrRB<5;OUE2f5E?c6>waTd%aEONdACc&!G=BXJ@63k!e49&?JHIt+Ul@QtT(j)dY1$d&k#-K%e5Khs~BFKj|>pn{woeT9= zpJRIESiggf(e<_sFc|(5skI!)Yovcgp6k_Fp|bRS+~Bhf&m~;2n}o> zEzu@9L&85$jdhUSGJ3NPq$b?cG+9>woz>E_U>o}+g%(ROqv3Y*VMixsZ$VjQ1W0Pz z0ddIE;X$Z=zQ6XTVRrehPZ}QNjB-F`KK{R5&u!>Q0%1DH) z=Tx+VzpE1pIXTqnFV@u5TnD18k{is!1#sNy{u}c@lf|B7!C{+t9i1Ujk?T>%Wy*Hz zP-vBT@&qA)jgp@%=MFDsvOatW{+T0_qmm^W77@`7cnhe|V#fYr*)1mCMJ~3P zM`%4@k@LTj&ZDkC+UdR+&F`?#iL*Ibg$06iS!DRLmt1o1>RfcFt`GzC7l&8!e}Q&; zd;8q_dISZxl?k9Lmt*8qmDONCVBnpFg#}R0LIXFDi%>p3?XAV7rO${*0s_L~e6WE( z5YB9x2_TPI%z;Xe_j~o~)!@5C&%H-B=b^X@9 zpW+UVPIE3GRbm3)Q}2(9IYdlOPGaA`&*hl~h>U~@jfK@VzXmj3QT?cBekv_<*JN=X z_tDexALtD<2j2oWrcJYvl9KZ1@#8k2sW~jaCkS5jZlQ{GiYX)BR$rW+7F-k8ivY=h z(%GMO9!_ao)mBN=LE>6naW7WRO8qo+w5e~H?N`y+lFYi1YhxD`h}yITSr;i1<-~V! zDfa59IGp{AyVyMlZAVF@z%7JBfo<}=vXTqf#p{No@knL&H6ez%)^R#5TbC_A z8&=_&fonqvG71lVt#@|M-ERrptNS^N!hu}eD@uK7;AYHAwh}OZeREdDZD;fy3pge;Fc! z8;YRr=p5>t9ltf>Ei#%8(Z_|XVV&*6Gbt;Mj|#mLv^1eyKd9K1)&n131rp9}&2HKr z?w)j}ahr1v6TVPrJ$=xsAn5O=!Cm35sG`V!x1tK7Lk~q@gGqY{Hwy1aeA(>RDC&bv z>_$(g%8B(}JCAF=^L|tCnuIhdyQ+?A8E+@1u%c~s=JR~^;@}$DLLjemh{?%Xziz4% z@y*$qT2u1|p05DgZODadz4db;WHSH?20f#t;VG5EVh%QPnld-XmrIJ*N~j=a{o|Uc zT?d!;yB%`AsSJbkT$@WJRx=^1?c8rI-<;A9#M!e1D>PMGvaIhsIGG!Fy_kc>ZRoV%BHg ze~*^T%=c5SJf?p9Bgx?x=`BvAB_uc%ApB2<`oWf3F>a+BBIT;^W83Mz88DQ0fXYB7ev?Yt) zLY!{;sjS~2!zKUxc3U2KzYagt39t2!gb*3*sMN1$o=K!6w7e@i_q~T8F5Px`{zAh- zxe)j6b?cDnuWa08VNA{$3=BdCbT0sfTh@ByLZ?h63=bm2l7Zrac#ju)3s2`)u~|O) zsul1|#!1UKXjD4)Li!wfy_lU@BsoLLqV0;J<7jZ|qFY^PTCW)dTNY9IXHp3E^i$bdz#l!iHVP~UKGi$IWq;VTHe6F84g;wNo~Ry(=d!3f z6hHLMEL&%K+Mf!)=b=AXP`Mn0<-Smwa>tRW5u@DCk+|H~SK_FY75#9g;Nf7-%Cl2G zq;eo4|6ZNcoNZ*aIw|m)%8AJ?FF|XzQC|z#p1c5FP1PEt7|%`>LdVUvNMP(OwzqfY zNK}L~tlb_j%b~cLuJ~@%5mIRwL>%;vo9NZgH}vmDQY$;IQbP>xZ; zR4PwUOt>6gsHJ zHHy+5A%H&JFXKfZ zLOwOjB`ta}2)~2S^$Bt+%AzQ9Ddbx?Z+ax>2Ql9nAXUeX-pd7B??=w9Zx*yTs&EI5 zfW)7a)4FSv-{w|-A>*-Z~ zaf?=KqoBaTyy&ebFQldJ7WfizwRY%fLFjH$94W*_x6!hga@wT-z`2)emvVO$!4>W> z+e?eRHTod_;$C4Fy`1qCK5rYr@&j>1h{4{TP}A-j5rEL@m;0IEb-V{9A`51VN3KVm zJyA_*tLzXu*r7HVHY39JnTUO5nW6=A5ll3X)ss+xE>hZeo@U^3%BZ7OhBuUGE!`t>d0x zI=y(^ggbn5g_f`zU<^>S(5Em6on=B&P=vup1s0JQUmw`@&N7cdy@$-s0x|=XQzx-g zp0N)a8H+J?5X1jD& z|AYDI=iVIF*xn+BsDR87HTHP=lg#>0V(Au$}ouIuVIVi3kDbdfDLbWryA+7stW?Othcvdid2TKhb3}@pL)<>uEAd z4|uH?6!O*QMMRySA8(>k9nK6#aB5ukT=|0{vKS0*a|lnzM*6Gc8@4ksGvA9O zeI^0?iRGb;NHSyZilN2*XL7VE^6~+pp;)tCJ0ifHDe3{W;pk`jwefk40tf7j@YF|%< z#KjTf(F&6TH$lq6!eUUA82Yc2Io#}ZQe=Y6G8->5+&w((1oic(2aqyK#Zgxa-M#tE z+mc@lwC9~VcN^y~=mg0TkJa@HA0!Qhyu{OEk8I6E+sTBqv@u`4aJYDNM^Q|Cbu`c5 zKu!^psvpHfMg}tM#-*i2g5*7lob$Hza3-%?MffV4-YX}Qab<+p8#{J`r%&4WDUvES zG_?k$%Elghy6qLM*;0&_GES#YBJKNuIu#HQzy^iu`e!xq>n~+AE_!{4S-RyF6*bd( zhlj^?5m1C;&v9Auh?wl_5%|?imjuZZVz!+;tA#B?>mNg1I0U`Bi_IzxDHa(MU_RIH)TpOXU@u0w{-N7FhB zxYH(G5hO)oQmNm(y}e(Kc1pn$3ybZ&E>CYwHe~0P+`bq#h?ZV{hqnZ8RnMd7aheal5fXzW& z@s7>RHrZwlf|Z!E&Xm0TIP%5k%Sk$Az$VK^n9I&vw}fP|_B&r=k_9SK zfdBg&c+8uFm%gWaYnMzecYO)H7z9Gz{k$NlKP|YLCWuW4OrZ^=(f*^qt`~O zDVSl<$+2i%23ab)=2fqy9z)fo)dtPD>gPFCg1TUstR zpQ*ZJhl?A^ca? zA4#PBX?|dn`IN1+fJ(HE@&Q!uWWWIZcWrp;ipAYDJ40{QmD?5ue?$J)`#e(&6<-Q_ z9hbaJYvtWPIGEqr`P0o@Y3Ds$J3KpEKIip@LcqQ@$S|mHsXs~o%D}7X3+bP#t*W|{ zr>B`pk%I>Rqp#H2bFYicPeV=Z76Ac4bH7;y<^$>iH===w-I&W|=iK}At} zxXKpiIEClM^{VI5dhmbv)xT?Oi6|%v-F_+}{x67j*qq>Ojx=`yJqC!#=x8ZZ)9m$v z=`W^%N;g9N%*K%s5u>fnHa3q#2%9t?B3Z5x1nsUl~=kAujgU69v>858Gn-LrioL|6p0opB73 zhinmja)bJ*qPcR;n-dkTdzobHMrgac{pU|#WsiMkQ_v4l)SZ!>{Az!3au_ysd*q>N zmWa1kl@7oCY$vF0R@hB)f%?RTA$7bnXnx9;3>~d=DGZ+S1lG2VoWt)a>iE%7O<*Nq z^!4?*9}Mzqao0ZPRHUTmC0a`_Ww}`cj=)=1qLc`p1KGHNbOFe(Jf&Za&#h3|l2}oc zyz=;b6_!rwGu1Zn>ymLS8#F?$bA|btBaON&IzB`1>A{W^oiCWkeouJpNf!A#dGy7~KuXTJe5Tj70C-!Zl@WI0;{NAX|6yS}7byF0Q zpC4;nvx|W~IdmQ2kS$TJj~O69FaWZb=vLfDV>>MMpNE}O?V=)j z#zgBrcPm}kLmf3V>#+g~e-`Td(Jq6jyz6O2hYY=m2?@Dw6^ioG<$VsTz+OB0EO)NB zX<1MrQvY`CJ!K(G@|)$O(_FcD4%7C#@wD-hf#IO{dhGwb16%J4APy%$<@f&2CT+wW zuAG-Fp&&!B&H@4{?@{r=^#WO|>Qji=zOdZ-(nIT|IH;LqX-xT2C`Q&EU{thn`g4DDui0Qh(#N>qhhQ?vnyBEQ z0M=T5r@yk7$91h*7yUoOBJS(Tq2>ebZU9wnNAK}UwwfY*C zMl;VtGkwo(gs)E*5j4#81OB6$L5+=GgBuFbGKqCW>IX>PJc~j+s6smj!-#BZ2Gc_e z@wVSa9!nnt>(Uyc#5I(3-n@Ck+p171lPCtQ*W9AowJ=3%s zvX6Js*yl*gMN0^W5LC-tI6r~&wFqj{>z@>fp(EwcS-%^ffi!-V;9vtvPgIfCx-Smt zTGP!@xnHT#;*Q@ZQ4*N0L<2!fmU@OQ=tKPYDHxm3+ zeDo0f5Bu^RnLBr=?j&%;(XO8C9Pj!~Oji{hvSH`+WI(OHARcUZcZ}f1jS!mkdBKLJfA#d_{Sge11oMKjCVLkQddyfm?m=Nl!4X;~V`H$3R z)Mx2USFIUCr1fX%Ui;3Kf^Cwa6aFYe&uM?G{su?IhOs9@;$`0u-s)DXxZ38l1j z4rxaK9R9)-iiaO%frSf!jD#U1q@|O&MBs3Fp8t074@moRXZuZoT0YL!>rlSA6lrA^ z-$=a_riD;=Z?^^|CF9eqHpCs9y=9xWFSU7I2M9%)iIzqWp1X+a*Jpe+Ws(~wM$q=- za<#_&7wUg9Uglm(o9a$|EUSpSIaBBAWf3!k>9lI&MOi!@K6;c~BSkUk{^RjT6H~kl zP8BVLlXuI)5lJ72Ry|{li>FlXd=GP7TK&jhW0&Wd;)nQVxhXd5B1EO{^V>ON<}Bu& zIv40v-SeA!M^6l^%PJVF2%q9pswYLG9gh_(;1o$zBa;yV&#K+f!Xk0)%AMaK)DkI7 z1uBZse2L1fI>qYRZf!Vknm%QskGipS)AtfTCOnl5Dd_GGlH0_Sn3;le%|?)@xM^qp zKn8lp?6&=K(0(rdTE)>U^Z2AF*|NZmvfW{E(@3OKb+2a3D@C@(uB>D(4tt-ZS~=;g zsg&5^9olcQ8~i*m-P3L8i{E|3H;FZq#yeCmN4KR-zweM7WBOrqs1TW+GBsV^llpZm zb*?D8Yar>#Df}W38$U^}0a?%MP1ixjt@>8zM`msAD(+LR>fzd-kG`5^1R{N56--o| z$|f>N>vAU6o$y0+1=@9*cNTRrGxZa`UKAZ(;u@osx58i0JZ7dS2C(v<)HUdT>~(lG zMf0`)hOc`^EiF($>SF@wmP&^Jtj)dtYNnH#J@eFv(R2Ed^f6}7XW?fx;Zy7U^eqv- z%bbU`bF`dU$$xMlRr>C{w)E&ddQvnJv^}IavcZm5zq#h{uvXVOVaqC{%v=`*S}=rh z+*_eFC*pEvxvqtReAzhBUh>wv1HWE(K8v-`=n8N&51YylR5=oL`iq0KTSsW505|*E z6)Tb`l6_Dzj*ZJA!|5{N*pqqB=pr|Bhd8oH8|$UCg*!2Vi)d9=(%-^FE+`YW@qSfn zG_1g$vX+@*hSa^Hk$iX?Yaxh5aIp@X=@#4PToGodie*t3H#9~SN4Xz1)uTBIr8WH;Q^b&Lis)`( z#n*XA7$nyhxHBeK6MnQ^_zrjZsq2^0UjwSl_dEUjAMS=vXeicyVOSa zX@8HqJgsJvT>0pF4&CIGc2v%RaaiKe4`>Z2T?|-rOJ(mjS3SS$$Tgm}sC^=)QC1th z$YlTPLrfp+!;x}d^K1HrHO1gZA5uKZZd_p;rCYOMdZInK=f(J!u{YtbJSn=%dR1eP zheF*E!T3=(_L}rGS9yqYKf>IZu@sd$%Ftxpn_B##ed2WE`r-wpB=cLl1YdK*ww>Sm zc_!{t%N9`aSg$t+GYh^(u0?+3+~o$DbK@;CA_TjRq+g)_2AXZ`A=d2sedX3;9bk4BZ;*T%7r{1wX%VBh$WU`kF}y zWS~18JqrIZCgzF#y(k$I=$E<6xqaW@(R}4)L@ ztUh_|w`lENo-apc>r`7Hk;uQkN(4Bp^hS+LcwB7eA57bn7cSJTK_4I_WvSMv+ zZf;d7i(fza^y!n;WCe|GoqhO__-!-a?Q7{(PLbZdQr((Y4?61>d_#A3Y_I+d`LX-h zZvQS7@H~FbXE&W=k4q)?3JcXi2#J|!pDn`SBK?d!uR zZ@X!{5PaA9XuMd^ZAZIipGg!(YzUP&fvgBRum9{y=Cv97mDC&`*bNT4xVZR{Qq=RW zxj6&Ta{IYP0q=9pGJ~c(Pod1r%$l=c>Y|Sb2^aLcckkx55fc$1Ruhb5Q~2!0?&>j2 zbp>G(Gy9Zbu?Y&@)G~Ap4l12Mik#Pekn)tdJK`dcR7&B~5RK%YhoU(;I$B#^Heu?#&{iE7#v zG&x(2ehSC!(H5thJu37#T6=_nA(0{G^LpfIl2#OgNoSD0I8Wce;NUwtRY8v#f$z>_ zC9{B#%f`p=0qDi!T2+#Ynr;f^T>}G(O2>Z3!*UGosiF#;t7|W3rKQb|CXfCT5aQ8; z+RoPFfB*jd(c{PM%^pYgUi!at&UR_0VAJT3$3=C0n+Th^4Ogag}^x*6jIeg&2lhMn`;n zm6Mg(+@d$0rM;|(O0P5f@#Qa0n36vu;$lksVZl#6#@Kbhu4Omc@g861gJ&&Ez}G2$m9N?q42K_E#%DX4 z=SQn+;ABCV+NebdnC@t$!QqeMQmf3(%&hDg@$pG}SP?X$*i1A&p;%IVNfq6bm8QYW zP1GH4xo1z7BgKn&tXX(mx72yn=~L?J+)KZK=j|=ykzX*J8Xo$OZS@TeX|3j(x6hKO zM(DQ7GB~YxIH{x$iCzBaWMhD233CkuHav0BYlc=9T`;kf=eR}c+bsU(&B#~ID6>B8 zb%-g+i*5)$570wyAPLHd9Kl^L;xnWf-Jgf&B#;t!cqE+copoU(|aORI-xM`qO*tfusiZg~z7RCw394L;^U!7#)9pl)B1q)p+ z9U0_1(AT}(&%=)*0!uPiFa;CqxG1IkTMnSgey$7OB<#M#7zJuHMW5I`!`FEL3r=g` z)n8qCOME2j&R)v(bZNb;-Rw_<*$UB<&DmFwDi>fzwFm>LS2V|Rc(By)kB*vI(^2%g z?_H_h-VOY@pDhXTiAEvs-N}@>6b06p-W)27c(!{pKCMeuu+0jN(Zq4b`#!S31VEnv0N%u z$+X(q(_Qt&b+P))H-A!TX7et0!h{Aw0M6yGxBEdphbdxh=VP1b@8JR#0kM%9L2prg?3TzF3o9CY9a>m$_+`&2 zU1y%zk{{LPqYRh^u2vlV>Z?2~t8jymgo^!RbHchW@Dluv4j6roQP<@HRSz;^5b0oP zq%@TFtN4M)mY_-B(^E0{h8q?xxvNcm(WheCdEc>mL{ZgNlfY78E8BsAzR;W0HtQ@+ zf~cBz(8+?_83ISnlgIPEIkd?y914eRM8?aB^wYN9ZT{}mwWq+>5rze0e-=*&LJszD zRaKWyTNhQ*xTJ58eso12aX9U-(X|C<2aCd)o;PN`c^LcW@uur+sU`ZuBB$Mf+G>4a z{A(D(#8hnFi6D5iigSH90$H^0JyA})xSGc>qqQ$Ks5nlb!UT2+5snFx5vMa&Cl<%r z);>ZMea^#WVwz|@QZH>`z}Mjid-QU!QkEUMIBU|&TdQ4K?A6Ql*YwaW>xvE35CXSY zc%P4Abfyiq--`am54Sd{FUndEzK@8GwYiO3oeqH`ObBuE4O9>Z+I4GC&TmNy^Ry)z zv@QuwyEEElX>bxs&HdjFajzrV4jkZp^Kx&vW9Q|z%;RksA+X@_%f+7iof%D^8?uE9 z|KO-ahsxKSy#K1{QmHQ-*#o8z`@#{@w+=q0nM`6z8zR-Mp}^(T)gFj7j^v(s$Ga2#E=)r2=_1*Dz5Gul!>#U zB7x5-Bp|n$z#*!%uy9#isObq6>(bR-^Y<`Lxh^jwaWOyUrx0>Hl`-o9=VvH?2`k1a z*#6RX@fO`+ntFAb@x(4@e|cn_R|?*H!zoe{ra!xBCW~_;md7(|Fvzd3F+5t=U-QAV z!42ZoiL8Kd_uy5Esu-+(cq{zDUjoF7EPbtay?WG+YIQ{+oIX}!d&rn82M#?R;`XH8;{N3k0 z@{ey~W1q}A4~cI4%0!ctlB#q-NILUQWi?lCk`QpL7~f3(v|3y$aCd%kD{F7P$qv5m z6jgPgoQev*#gD{K0Dp9Lb(sJ}vSq1bXlSVY%?jVp`=HC|Ojt;0Y0iD2YdBl3(q@9) z`y3HmCpz(aSD`h~TsRtOmhp6hYD{{6Etbd5f!gUPv?3a1(`TJ_)3d#~q8@w_l1G-7 zmMByZ6&1C5dT?+cp{-4ln%b{?@ABdz%j;}!>kNiXUPLLctD7vCY=@t7zY$A|_pBsX z3#QlYprNB$X3!t+-S4z3?bZeA1N!C1j~_>7Zf;9;(T^_wN`o;19Ys3g&}gS!o3knqSuLE3P<2gd<)UxaQE6%6T&CUB zA|lWB_V(iTP}OB;u1U*oe*U6Nwg|h4fa4uVnIhwL^cS`7OWd7{p8W2|6xHKXU7f}R z@!iXwNH3C;PeZs+my?&*J~TA6u}=KtNx@>GfFndUlG5%ogKlc-cF_5e7W#1e0Jb0L zyy5;!#iW9|ur8?8Dp_=h44+_1Fe*girF3YHT27_ zm)Teo<;@1bk|k@yyElg?68AkPdNL@nem4p7GQ?b}I=UYXyF9Z57cZp?S%G&;?E1vA zTwkrYSU;@<Ti?LowzwTqmO35}pntXl9v!~_c-z{RHa}R%%k9W6HuZBJJ&Nrebi?-Ng+Pww z>QpBxTzehWwZu!fjtkh=D?qfBnm{);^3RX7oA@>2PB+LF3*DWq)4a=U9>{1{^62d6 z;i<5hLN``k+21w1Ig`*BG5vjQcVZu*@Xcl#@5~+f{d1z(${O+3+%rg#3>I$M)&7Nm ze(%qorbzdo(Cl3-Esf)=FMRd3kNP=IQTwrh{fwRUg<9tmJ};2tl%!pWf)A*0Vzn7m zp&5?sJEcSAJsK=ro){aSw!;i5tA!Gp8)&&5j+O>p3$9Mpr5!6OIXs31C$;<;#IL!K z*VGaep67%_GaARm z9Cq`|Ijrh_A}fm?Sv{2%J90+TGuzr0LY#PYw^wcGFm4!%h(yxmE%L>FV9SvW)dL5~j6jOW<{uNKp z8BVcOIoALF5;U;j5HtOjh_*7}F!meDM84;5_t&&zEzxT=#aBz5Aysd%a0%-@ju^m6 z9tlo_jyc591{e1q9F+viHS%ell(C~~oW_#M;C(ZBi_s{TW~8z3H@kjBQU7bIXWIH@ zq6~kB1~#XN_0*qYkFC@Qdp41S*_Yre<(4u}%&Qx02cT@h?Z|%G=P+J;D;Q?;r>jE) z$8~#SjxLnp0U?hw=Ggedklse{is44I_vd2l6dYnCVSMx0ARxN7hpX^CYYtT>1X0&z zyB%3T??iFP4~Z;5TUxnW3OqahWxgkg#p8VK4u}UGLYN%e7etZPjnAV$un$9+-EM0R zaSs9f9IsJAWkS?8-zP3dkfv82R~}Yg_nqor5o(nAIFQq&4r%&HL8@@DBAZ-J`+1vB zcJX=LZe7|mJ^6c>MaQOyZ^E|_rtz#aRb*fZb%zFrpU=X^=;;5@b&!Y(4 znQzfMA-ji$hS=E&J6P_a<>s!M6$5`BVPXcv#Kf$vCx@nXo+gyz7A@S0!HSDtf1k*v zq}-i;W^9? z;@^7yu-48Wqvc+a-@S3|FUnFPBqJb9Y1f5N@dE)h?)a&ysxM1ZT5|GZadGh&@w(c- zaZgwKcRXEgeREr+%AgX}$V06%BI7|ylmn$LFmRIw#IYzJEcidQK6*ylSI#9DtjH>dvjxPpM1ccrlEOn6xxfJM%@&W-R z5C(e(6soO*&C&ej*#>9pG6)L`rt9`^87Q<~S7UW=&v4Uukp7P(&YDb5@m6`{*)jhN zC@cHRNvmFX9@K-#y7jj-QS^n*)l~`)$f+ghoT=QUV8!Y1$&hnrt%Er~Tve7&z1}P0 zg9yyJ>|e-z=^NIZPQZ2ivc@`G6xH& zm$&!-Az2#c-12^9Z*ZV_ek@lmtY~sWC>PQjLt|;cZS*BAE@L~W>k)FQ@An7yiDgXd zpJaI-E0Wvk2oo{)>%oWc$3ehx`88gVFOBxMb$UGI=7gsC5|^d^b>!CaGjkNbj;<>%T>Nm!A9zIjrz zNBvTz{op}Y`V)O)kgE*d<3#%19Ay-l!SBOjMU7g$1r2@wn5%sY-{4b2uaWQ_#@295 zfFEZIirA+W5D^gtS*CkDG-z}|^y!)LNr+*CY#t}9Op^lgi3?cddA#unoyz-dQ1BC$ zX3vu%W;1y+_EGeNSepeOU62l@*akIR6WH`%Xy`Zv1MQ6s4TzCkMeB{vz50zcR)J!l zRLev$VlHa^t7+Ss@sd*3ZYOqU8wi1HgFHU&w9N2<0l4*7M@PpKJKCa=BPo9eb{5=i z91?dyQ%I3Pa>w$Y^$ac1h{;Lq9~t6QYA z3W}_>zp0Al<>n@+wir}R`Qe3^6m!M@9K=vkRKUItNW(iXg5`i%!{xrGj{|5TE7K6z&Y;4#ZEyvMJ)3PpdMf#Ut z`2@a17nX5>je(QsI2qog!dbM+jDZ5@;Oe@XioUsaKzU;|@|Vtu7`)f}MR3n_jm>be z7Kitr)r76H`4+F`;__B*ux}B-ZY{jEltV9PNkK?(6A!L?Tqr;%T3lYvwpR>^ z24WtSX-`DyNjE;}a-SHR#*h^I(;6#D{+1LeGvXI#B0%6(A$ykPxZJ%Z25Etgv9M3H z-Yk-)wxUq)+E+mlKqB(`qrHJrY#l?SMJpSIGC7RT{H2Y+@6rLV$Rf*Q()zpqgY%2+ z@=5@FYeby3#tNZ3l|8`OknFyiwOswn=@zC$S3kWM?-#S&W8k*QYe+oP?l2YdDEcHJ z0X!Q&rSJK|(FaSvj8W$r4a30|6s)KUrBm3K+DpKc#@{RqcSpm*w1WW!XMAcMTi^P4 z4h|19K28merH=YoP(Rn0(MurTa9YwVS#pCSv9CjFO8W9#Fp$!1eBlRuLm~Bs+9)J! z`?Mp#HsV-pq`HjO>6Wx5MbfOAgwo6zm1EzEyR%Mp1C#RFk<=;K{WavCR3W5-=j@Yq&(8JPf}EArb0N1K zI0hjDH3!F#{vZPj1E9sYRzY?>JPrc8pXUc@s+d6TZcyJ1Bjv%Ppa`$87br1o(Y zq*Robf2$`geLvmr#>2HuztCYgOJ=gkodbtTti;e}FqwDFif`&;NQ+((&tQVR>3%=~ zUlP5D+x~p1CrY(2-} zG&`$*{^w6p_LLb~j{aq*7+VDb#7kphHp;s&opvctT$I#ZcYS?5?$f<9+U((ss>oKu zSr5E@d=Ll(Dl1fsoJCP84yB8ZGY}U-2-Ge>RaUVBRaw1Oz3N?5SJ7#M_n)e*I3Ow` zekAb#P5wK$@tT{L@*Ysq`RKu!nI<}!)h#4z2HI>uz*s8O$GZ7qL%52JBVi?<-EGEZY?e8aPSy^8z1uE{- zh>II3K_kg{i)O`~yjJ_;iqNY7N#_-z?@;H~r>`?FlcKx?4VPb;Qs~YvQf%eFx}&)+ zE%vA7JMfmj1UybhRPERNz@0nz-%+ywphRb9(*o{T!fwQZegp1WVQ!sbJgZ8|7saXm zz{b}l;y`FW-^h*}y9S&arOytxc)7c5jVTy#7{K!3-=d@GwT>PiaDZ#xXya->ZMFL_ z=|w^VZdMh8ZxtrS=G5d(g4g2G##i71*3W4a zV6YDzLm0%z_5@MmsJ2tod{3B{KQd$COwNZGU8G@$7zIC9h>*$a(%tXCf~}SPPbYrQ z;Q5y|?S~Y7(DQ%{#}%yNeFD+|uw!X@HQM|$qR&JzUWophdmynEqVGs* zZj#xb0T4~0|!N|iS6HM`i@yMd>R;?el-K@SGBuWpWf>SJTd-ACc9k2?Yi2yVh>D=r$u zpdL^!Te0M__+xuPz`rf+Na`XO zmXXg<@ZCi2uQkk@Vut~{0*T3m?$vOYE^u=H;uy4>i?@%&{A%mAJi>0CKcu?)^07UO z+><|Co)~m!urJVgzjx~daAw7D%wOLKbhFKcsDa)sux$zqWV|?ZaBk1Sn5%H`)8s&1K{KAcPwehCXb71T<(U@3`BB zR@hii@*ynfLkF(hD9UYp;MIGopi_a|`Rk?kW$;~4UmmDw?sT!h(&c}iNVcTNnsH;{ zX6X{lme4?Me^HG-M1MD!K1$MHC48FES^~^_?T?U957mE0f$T@(Z0#KTXQhxdnK91a z-qa)9**EEDK+fd;B=#v>!>2^9@!SM~i$QYfh(*9t1rDGB=sI2!ys4OQ-YcuPXv}L% zm0gzof;3}Ss0$cJ2#jOk1DdVw$V{~cZFB7&Cxc%j`g-}=<{WEQ6Y9+Yd3$Br2d_Dr z|2RB)_r|cZ0=YZsvXTmHFXDosMk;Y&D85Mss`r?c9pJ<(Nnq6W zUr6c^l_5<-cslPRQ?K*oj*K_&j(dQRex=cpBBRa*9{x#IFYxYWl3n}I3^%p!hnk|N z_*_&VIe&dl0xNMxK(8d$_9Pa$CGOvaXsyl>YW>GP$I?-OLx%x8{ovsJ#%Dh<3YcJ| zTW9);A_-Y5wo35NEnxQy=#jC$P7`tkYG6D{0Ag#IMP3e_A%Zw?s98Xs$-|EWw&@hS z+<$n!^~X2S!$?oMJN-KnxqD83kOf?$`oAGu37+ErT6tJ`cnASRkO5_A8052^*@jV2 zc}C)5qzsyoFNM7YTgnkor{OdiHCi!bOu-Jg6rQX+4KKQ$(+1K zaulqa>y$IZis~FscXZ6m%}r)%?Gm}oKTj*C?{99tkFX%riM?QtNEB`)6iXT-B-J`X zBGJW{V|GKps~dqbnUkAKLrwklxVEjWt#K&P2egk7ySw2i2n()7NA-|cD5wsv7UVox z|Bb%E-iu*H{oKQS|2e!G-2`aNKc*a>D6?Nau;eZWVOhlkSZIHl5w$25nATxsRX?KQ z4y*CI^&KlWY8w$GHJNCQ?C^bLbbL;FstS6T}Q)J*5|n1mgrK{svK1 zRekj#{~xKdz-PsG%JXcmbT1*~s48Yt`t((+e)#_Qz2WMIkly^e7Nvq8lZwxlU$KMW z-UKDQy}f-*8u~9x?qBN`6bk)oVuBbz0JRRCo(KxH%WGr@F6gpy$yJAxNDg$LXng7t z7yb1n?X7?gr6nYyMs`6k%DJ5fmmkwx?11I^CQIewWuZf)jlo7!6uJM zeX8C^Ik78J#oBaf{Dfmwpc>*$70`San|$Ggw`a5PX@&kQ&7X={ZEWSZ^YlFiY11~9b>xsig_>Ukvw%#PefY4uk)=;&(Y)LTd$A_k^}vhTxqFMMWR zXo^N>W)6;tJf31;>rny|eHyQDu;qT5nRH|>Q46|qX<_@EU>q}J6BJg;)CM5#f+A-Z z-61_Y#VSoDl@7Vk+ydZRfEnf z9qr$1Mgoo{v#=!~CIm)KNwYw9PsJac_xlh_nQTc=VuO1k z8dnDgpU>&0B@s zhuq)ByA}c@vhe?YkC&4mqtSzq*yvk9#J$JZ5+k$K>5a2xV^h)FQP2iKa$ljs=)=RN ze3KX!rjZZdp7DEdJ%?0z65uSi-y3tIue-BArKDx9+8QDn0JK{TvGUANS`qrU>JNXChbH}0t^2%8Hm-X7o^G2 zHsMwc^YVrg_}tDcfl3X(nBIyHnXMHz8W3Z*DavceI!VEL!?4L4*hC?U8RY;>;A<>y zt$ehgtRf{A;CA&M#1rU3s3P?u^P-VaEZmgxed=|XpP4h9;uUW-Q}day6S!$;Ke=qz zrc?*$paF&^?UJzab$4y2fbUkeTU72I-kUS|k!KMFmns*JE_h~-7LR!@G_*WB##eckEbTRes(O?sn z|78fcf3n_@);}Ns|H+d;FnqH)_pCZm#Dm-0+xu6gsa#9SyQhFn)vu>}q1X;e=|i*! zdGhzI93CG20 z`kx-LUv7JWoDsa=c1sCbjX-HdNw@d}1X=E*udP8%xBB(1~1qH^D}DeA}DwO)D)ob ztHfwP$6~hE zU*GNLJn6W(z8r3H{{Xz^G*TU(iA9>_z4!@zXNd<;UGk&#A-TE6Q|H7Ojt976=U=x} zpoCrWoI7%E?Q*rUd;p=`z0OrI&U6w%XoBjG+HTR#>{O?`<=^i zNji*DRCj+ZF)lx!5eAO+G-iaESp1x-@o44>1a`9W?bohLM*FYT{yKo>m8 zg^0c*JMbTp{C9&DZJ^$9d8|rSkKd&I{_*Y%F$%V{l!GzE%{jW5=npx?nx(1>(&jiX zQN7ic6m!+8#@F$TKu*!|QHU1A_0nmyRvsB09e*XH6V0YrEf?`D#H0h3v~yjro`4#L zANnsx;-iA5s?0GHMxwj^eQG|7ESe{N)r?he*kStH`=ax&>gCs?88vR-WPx(zcy+Qm0r@5 zphgRTt_gIEg6IRuqU%?~-9QMUV_+ycqp6~c@Z|s4ONN4Y=O+_J8V$HKyxBP1(VY0oq9;!%{jMA}g5~>f6qTjx8yn+WTaT@B^uxZShmToB zj}S`7qo^MmcNiTF^@H<%&&%Dqfw@*Som*jSnTGoM*c@%#u@jN6(TAx25 Date: Sat, 1 Jul 2023 23:28:19 +0800 Subject: [PATCH 030/337] Delete time(PO).png --- picture/time(PO).png | Bin 29177 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 picture/time(PO).png diff --git a/picture/time(PO).png b/picture/time(PO).png deleted file mode 100644 index 81fee6ef1b683dbcc8e0166613c2743cc40db255..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 29177 zcmce;XIPW{w!RKEd=ggV+oNL~hFTW3P5t2LiUVH7e*1kWfD$5WNP!IqB03tbANp%1K>n8w! zBYG1L^Aom*Pn`h(T7aCSgr-~K_S|47odtC9Oh(}nhw`3koQ}(Sq+oQ7VMD&cbaom? z;0pOiQ&g#-)N}F*k!@OPA=`n@&c&L^KuMJ{b^|Ca|HqC5ycXHF95@P3OjM4?CU9mo z`+4-@@((_L@|%5CK4gSwE{T7}9VN#Vub549i6+gzEmtyi0hP0jn5Q{u&WfM`dZeS4 zftMK~o+pX`fB;{Ln0Jq}?-l&&;tJl9AaWqWIgx||urItUKpL81kpcjjG5}lvEebpD|PuSesvq*ZaovgPgLUiw}Hbee6fQmz+S}KpsT@S0Kg#*X~VYQeTBoT z3ILST`e6gUU}0X&@Ae%&zdMQirpDIY?k&eWpWAQ&?bPM~@92*W>B9iP^9Z$CfL{iT zNgn_>x$*lCZl(eN#^Qf}3s`_IKL7zh9Ph?Y?68@qSzkQ_8aTSe9w{{Fs;?QBZUwOM z+JEg8umz8q)nCE!zW}k2X=_n@L zkB+8%3Nf0<5>M_4dy>MuFR6y}`*WKUP?mIZ-8>glAUqd8z5vAxK1;fE{pppTqM|D{ zvw3VOJQ^n?Ck&pCKFjkJN^^>07~eRF?UEJU8_&2!LsJQD^^w1HMA)yMa4VJ2 zY>LytcXHz(puP z_dPUt?JPpeM1q3sO`+Aj39`Ig-xe=shpelj`FbLQ1R156UFocZZIvd|57?P^d{`iI z3kh%wSCW9-0TtX=8y6O{u{2E1TG@Fzc9bi2NVALK2;H6^eEkWQI4V1{3lVET{s?6! zAdBPHsycO?PYGwW(9gI7KsuyC-r3RKgQi$0e$IpG?pCVleDX@#NyZ7~EnAj;1$T0m zk0}iI3;TR_GvNb0Gc`u|;H?{bb)i5F4P%AEivk@;+468})qIti$sMlYHOon+9N{5= zO8@}+vhE>>c2_?uYPV49w0Z--(cXUZ z2+ecgmO0`a@E-06)E3*$AJ(wQTeiFb14R*%$w(u2^|`3EO--6LO_~XN3N5`R45|{q zX+|a8tkiS$H5If4*oHjMI}p(AwWO8%El2N^2ok@5_oUHZ%Q#I`OZ{Y`+`5$7d4P@r>pNJ^k5Kj{u=$H6S-vGezWifSVh z+gvWuLX5qC4Oyn zGgW|P$n@9Q>RW0NI;Cv6{)!J<^4|A~5p8o1oWp#l zc;>8q#8oAZdXkCop!S<-JX<%e2ja|#assO1;yCLWBC}*MrjAQ9@``c#t|wkQ!elRJHfIzE&Y%AH z8HTC~|Hjbmv0fe3pvBWKB;@hTmVd^EEloWST{-HrsbFU`75(^lE21xK?mkyou8BpX z2K;%3I~rN6Vo;%!S-F|2$0fHMaUPZvKL>q_o3gPonQ!HloiEbLnOrxc9qvBOuqYcX z=7U<_@b$Vx=_CVvo3{>M-}0t0*8ssA2Z|K}^w*j19E-}Z^9ykaxS>BctJ3oS1$G!* z`jIv+?!U~tJJ%)4g2>3j+9q3V!*!8b2pVaLQ=j{)&W)0m0?B`DXvn%L&cd^V4G4d% zR0{xX-}(m({u_fZ_{Q}&eESDnDZQ}+8({&qZ~w-yZ#~2S`=jggA9(o}no?N&s{g+L zV7lNL1Jx11(N)0fJBb9;7k+a=**~>3orvdbp+A z?kvT!xxi@{qQBu=P-FY*#p_Yc4O`9n?#odtEhh2J#hhO;M@>4(F8oh|H&L3;qnlGY zZO;7F%}k)s<7$aZ7vE}5bZ(Z2^~kdVzWvbwG};XV!s~q}8e2C?>J$c4G?G#@oP9={ zrTI5evjvC@Zns3VJ9H0N})!u)p`F2f)CYaJ=A;yyoEJLRt-+x-|2z&*2ZFEDG zifm4CR&J*AHh?nq7V>)-4eAO?pi`UuX!ShMpCeqTRNKK)*UF+0SM_rsQ7X<_*rP{! z4)b=R{hb;1H70C_<#_?7tm_EP{}ooO7tGq)v-X~ubWn*H1p!Ex{(uAMs!#Y7;`DoR z%)jwq1*a-p))05-KLvzOHD!u+u4?$Bw|5f`Qos{JK#$Rqnf!L!xFxLzfEl&((}Rph zLWgGvGxtl>S#fV5{gHR75-0{C)WC5IKozT!A_lK-*=xR9QnkV`hV2Kx3(p+GFQ`PJ zm^zvyx$(cQ)W30?-$e~=n=kIf224KsUFpwbXaI}rFEA&-GvF`Mas4s^lK>(6`EQYB z^lRwxk>>>DfV-}#SVwA)cSGcH0N+KVe0_;WhB4lVLs?y0p$LM=WrhTBz~5DeTQF)40oW9*Qy#O)1=TixFcb z4r?;)-8tUj)2ccho9E}>D$TPw2Ir~3Us7~DcNLYjU@FuSaCOkjy~pL_gIKQxvCXk& zFCv4z%RIz;r#=;aCQ=y5UF1z1y|G3UbsAkS!b<9@|0G7EY&es!57!dVbqmXBtJAaE z+!X?!8hRJDYm)RuThM@U7F*oo#?KY1z5?ctI`LH&CN-4q zcrrVG0{i>)pItT~ysfCoG9HV4Xof|z)M|~|W0-fTSR;Ak=FEG&mm$K%azW&8du44s z3h35rT{G0D9a_Uu0o5y*>M;sg{n0qQk~;e(?Xk{u(MQYooKgd>PR=R@$z=VNA85eO zYje+Bq>5bGni9Ou-LYOr4#Og7M9~rdCf%=by|)w%>OC0Jaif^mFXW0=-D}wM6&R&pN-l;v&BkhK7S%wVuC~6bwQ6L-?~7nrlk=)lF*H#q7`&}gY;oz4 zJOuozR>Psk5cmIf=NlQ*1^{3me^-kl93LRyaeaPI{#%#)^Sb>XPXV82K+K>Su6bvP z@fY@)nk@Z~2WUrZ4~0$`9q;t>g&PacxIYZe=z$(A<0X?~GkqNs`sJSNQ9GQ!UH=^4 za?Vk}R-tM>Mt&=1v>H+?WqBx1R;|Yvj9uVWwuKG&so6hh+0(TfI>EG8`3&Bvs-G7E za$RZL+J62G=OT=PsT99+$7&;l9X;UlzQAEwJ|>&hIX~_D%NsvQ9Sp;y_{ZXX%JM6J zBzY8^_Z?ki)BN1afo3MqQ(n(2aHfqIDml-qW*0f1zttz-ND$*juZPYR4H`zv zD3zh!GbCA^7o-O&ODkPd@F|~BKF%L%3^oucYEQ_8;HaZOEy%ekL_e8R=!s8h$wF0o z(kfCplf2)`Bwiyso+-sJU)>oVdz0YVQ@?CgTdyn*z7ms3x3a15Hu6hZjih{PeHeZO zTQSz;W*cy7wxw&X`_(Chwi@~L#-)|Rn*zeMrUd2^dVC{Go=6Jqw_&zpWhEQA18++C zE3)U~6WM}OMpoxA6))E_Y~DVwJj_)RX$tMA)6aiZ?94^=ysE|brgw?ew`j;iODFg*SegMa~m0~IVeN;<8_rk zfM;`8lxPvJ>gcZW4!56Txzyk!_<2hbRWhB&`5K~U_r{fgn)e%k7RiLbYHot^uEt@i z>x(n#C{Q1_;<4zq7TOF}Cja2FXo0!ldE+n8HuW?t)9n`Bc@}8a^a|#Z#&PR51GjA} zp?t@6)|7Q!tHN9EC&k}tTwi=M`;8Kn>K8Y@{>kzVsXi3!l_6NwmS{C%9p|PpEwyH7 zZYO{tJ|w$dM18)`szw=TF8Z9JH$noa2L*4Ba3Em)J#o%_I$c?sfAEH?~{B_cd#*%qmM++A%RquM5F?Iyc@Ob zdDcQdS%`6Da8)8>74~n$rr{B6M{V37Y9N64C~2Q}qbI+oO-mC+V=zL%o)03|#F`5K|6O|~d~OTQ(zNMyo50n9eUDVn9{|iy!27Di#2J9}x#g zmFvog3~CUAbz=<3P4PQz^0IFQZGH?IBekf5*f)!8j1n+E@uQ~m!=`*V>mtPx=Kz^^ zmIIK60SK%&wH^qb4~+UBhi#Z!y4U=-abfNt?)}e_XuI)PRB!_$_CG zjfNs%Fd5>RG>)TWP2J?JY7u9(lOvE|Y@0^e_kzmVvz_*n?7WL=h#)M~+V(`}0HuUR ze`|axckUuE813HC!)Lha^kC0bS7raTL~!stquTT!EpQNqTs}ecNkz z;_n#BI7(o4J{Q7!p>rR08KozZ*@i$A2dhCA^$tLFIqBGF=mF8-+d-SUn|2V}uAf%z zd^+QdHOi0#M`zQM7&fYJLbWl)`CCsV?}=rW6f(YhSnB_Bq6 zBC%CtVSfj5mf}Z(mZ|;x%^lwqPCYm4SmEXHYCU{9_{1+K44|8-3p<+G5bH6oNuXpU?52%RD(xwE9j}(trE5|j-9#yXKy*CIfx#khnt$IeQcE zVt7w!+5BA0S@I94(~<0~Y+YR#?srAG2!`Rm;{<@#^^ZXt!-CoWi*xgTVB;^73(5e1 zaGc+CUONmIGe=*a|2OC8hcyN2`PC+kTLFbei?Vr|P|?KJSS|{MhWdW=n$hC6?;|YI zP(X-rRk&Mn&g0@r7Rp_#+1Px7HkB8bjbMMIlEuks0;Agac*asCOS4jP2zv@O$UsynDG}+Ksww&!5iQa81gT+w^5mjd0Y(`$zf1!4A)ouluRU z$q|gK+ia)|cA~Lo>%sw`qQN#Fa@rro>evlkBg#^{1DPP-uD6{-1T@2%C+E&l9p0m) zy(;FzbwwcribJn?EOp<`8TOxKRJ%V4AGJVCtJb|ZLhfN|KVup#=W{+3Da)GOvru10 z>Z%q$LWLC3FP%aX2!FvtEe6DCp!@SUS zt4Dkg>*@^q{oDLI8_;EEFiR0QEmb`V>~W-wbScix#u+k39G+znD7zls4d;!CtsOy6 zth_Wp$sldB8woOcMXY57nN%wKxI-gE;qS(THXYeMp46_R8lz;(gI}-MGoO{MyLNfz z6wB*H0!0jKjR5%XB_ySWsu&Fpz7ZxvS&{jsTogL$mYw_}p`tbLr;V6mdR207W<~OW zVfsXOfw)4iNZIz}2u)lwy-Bg0i~0{Ljw;;sUp z_+ZQUQ<$qc5h>K*bS4nKN>@rGgbF!gAV5mjS>I>B@>PGMJMSRJ$z(BJ>qtz=1F#hq z)kAFaNK5JwyPONAjDD}!L}+a1Qwr74z5-c`a4XuH`c(@77Jslgimijww#X`IL){=d)H>f`ThoJ`JBq0Cb z7XZJxr~h^n{)?pH+IYCGs5%Nlz@ori@#{hVdPtFpxIX_rqPMo<{q#q4e*VlZxL1=L zbnQme;t-`q9NL35DFA@g7no7s{^(k&`(vK@@4b&w%-f(1|2-cSVx$HEkL&Y~*ZcEp z{Xd(JSaH^C4Pw-**q5KmN5%QmWvN@7*yHK-n6vF{@*z_q{G-x7e@Oa~*kZI?t@#5E z@of2sj)ywd!3V7tjL$E>?bXl?Rq))ENov?h5o$Jq{SvTB{TuYL*3ts&_T-b-O6|!k z>XHpF-@NKMs7d*5rYl(GqUYx=2CKI|F4oj7$$a3R!B8?9*Q7U?HhbZP5}Z17wI=>O zVs)utq>GZWE%j|~P;VuatNdb9q#s?$wKf=&uk9?u(WuP!{pGSSpJO7%PLh3YYp4LW z-f(^Ju@f_GWv$DjzZFDub(~{4O~W)}Txv(&fKuXp7LGJd(YK?VDvW(v#6dew)1A$^ z#07qrbrz3~N%$P&;)pOZN!DA$3wE+9Tn+Nza!0#)c^!SNEegqL zewVxjJQ$tmA!(pd8m$q%Hac1vXI_;oMo6e$NK46jb@y3GB^Q9~3@z7E zqPl4f6Yc%HH~F>E;Vit!f)b0zfMhq9c|3F3yLj(=%tfT~TdDkCT8`1Bbb1_Z32~HM zM?v>DdV#|6G`#W5CD`ojh<40H@YO%aj*#W}wAv#mtrYWGh4zu&u?FW%st~S=cTMgVvFZnGP&l zpQ)*p2+E1PU;%A58^T(CPFR)0_rd7TUu?bKP*$BwMB2Zb8DOe@ieMSeUmo_oaa_!f z{&{h3G-bGaL>QZM8cv=51EtuYo}zC(7NbS;L(3<#u=2y+;5uad)~Ap51$YZ(b8^7o z-0^$hv5Mo8B=dzb`kYmKse5$eA^$}CbMSJx;WC^=P~)o3Gx+oFVNI`IVfJ(Qd;yrT z1={P7P8jX-yt8dj+kV--Ej6%Rg3NOX0Mu|!_RMFw-$H8#Z_`>w!oYmO4N)I?S_VU` z`@1n0ySRv#0K9OueAPv0Sjhl7^uQu7MZDAP}{ zKU0j8_k?I&?yzaBBhAVy2+5!_4kLQh60oKORoWDO!IKYF2F*qaWQWwO-=}NJMujf> zc$PSG>&^tewHp03r}Dj@=Cav3CyW(Ai7Jt(DG3#y(0 z<(-j~F96cSXXb_G8hRY?SY!%*tO~a!AmHuDMeXVq-~N#_ojsV%>9T^t%(RA7$6uR` z&V-r=dC;yt7N4?~WU9`fK5guT7Eth7xOdJ%3au*7Lh)%KT6efyH^F%`<*MW2mvD7z zdz~-LB`;HkJ3#iQI_P)2TZxM%RS6x$*?|&n$qm@3lM5eGf}}z-~bC;S0|%ro2DeS10`G zE|0y>QYC$*=|FN#Pu*6RR9lQj#Z#i4yeU5lJsr5|)2)nusPRfWBH6~&mViQ3@G5d_ zz5ktjVeH10cUSq|)zO9X`?X9a;q!2t3_N*^asHnP;3q78P)g^h@hkn#LN)Rhi77*% z-9;4mhDP7c2HPVNQ@#ijs~fxSX9&%0S8E@FHb$p>|BMNygeJpIsB!Ek9d4yobjL9&DB>0{ zN^;oy?B`y~V%eApSkHbcUNJr^VX$$MRHfDOByO!>&6_2~5DNb?ZUE}yu?spcPT!j7 zfNqVJ>GH?5zRNSB;f0%^@@=$U?dRMYx!4D*-o*X2G5S_vAWa^jKL4R-Dd*D-)`}Ei zi#mk|gGJ5(NEC?m!O#|qE)rbP_!gE0M0UMwl8`>vCvr@V+IrYx1h}Jj2W#{y%NF@V zB8!2#f@$IS2#l^2WaHwr(4h!>zkpmK)O(4Tnk;!H zs|eL&77&6J`VJ+r?G|9X@dJOBTfCHHr|JI8$rHxFy~B77b()DI)ZiL#&5FiO%&o4o z6su+R*kXEnePD%mx~I4sR0j6MKY=WTLQjZ3mRq6}2AgB~BDN z3gjKvrmI90AACq)Dq1zSF}z!FXC*!Yt&z$EMd^9 zMAtM`%Bl8BdwOT3H|cTvGWh;|>4I^NQ4_~!{7qwEe&4ilZG=Tvrkv9orDuP$(fQ#Q z|ItR@7{8j8oJMfjcnml+!2Eq-0IHl(EFKR^8K+6#qKt(5uaw+3)4eAgGH@V@DTdD- z9H6^dU1z@vl#7-l?kBZ;$MsLg7y&kPZ&DRLJ8u%dGH5KBA~@-_#rWxU%)oC`3};$l zt(`P`7h=$8w2Qlht@hDzFg&^ZOS%zuN05 zLRZI((BW7D%J_|`E~*W>Tz8FKwtkfBybVB0N)NxydQDo;9;m&4J~eRu5)nP&su;fA z2>AFBw~FrX!1bZOc^w) zdPzVj_-3lC)>iPE`{t5-!B=NSBaG!XB;LsM5Qgnim9(?g>6!D)PQC)Ra2JVXeG7|p z#ta$wy6~~B0?VAovtTHN2U=Oz_Dknhug-{W!+1g9TG$c1YWk!5kxg9zi6oXaJ{D;Q z3;#+|@vuutM~U&B6&uHAG2_A((Ttxght*o`Vi}TXTs41gkD79hUP}#HuGd!F8J%DD zmkUca4EI;kFyW=3)jAg5L@F6>iYKbp>u|MnYx4U-b?2Tbsb6i~xe(}OG)r@sl?Ssp zxI@Aj@(qI7>FQhwSswrjKLB)js6~phKPy(O+G}kMV-;}>BTf6ixfJG&Jml>-B!M~= z=l45rh2MTE$5ysmVtS!A(|B8Ev(nwBdy>uHk%1Au-jcy4I93Jws1yXLJG_YO!l(vB zCs=^)f2anu2;HDC@`N2GDv=*bVfCwSMVoJmz+9eH3?+&hYenrpphW>|zg2_$mN~g+ z1DY5G6KNPlsE0dd0kUmsbjTTKRTwd*>}hde>3uaD&?@W5yPu0xHb{yOLA_~oAp{7X zzMa5z<8ta(llfG)m{;deVkw^yNuOov^`m0a@t>rWqD)EI_+6JaV@cxrw2kWt4+`h| z!@1Bqd!fDhv7JAoI@^$I@mNwUgEUq1-6?m0Sk_p)KhxKB(;hw4FmwwInnRjqfN~qw+C z;UjikV+gZ$j(TKq9~ta62XwT0!n}=I;IMbK>xNALqo0k?ANKT$AVJOS!w|!JM24rm z4n8_Bsz!Dqmb-x^{uQmV^Vev1b~8+N+`Hr09$2*SUf9qme9Rm({by8~R`lc(=c-;GK1Ir}A>xw|r;UfNgm)Z@%Lgyp% zw~^Uo(LXGEbQYqPba}hVP@6*Q<{%TILFx9-l7Q;8ql0A`m8)e;WbhDmhA@c32l&l; zpC2(#puLf3;*YwR^vUa*|JG#v@1&qPf%~;Ibf0pRe-M;?Ar@RvfEG(I=RbNOh9?c69 zv-r!Xq&Kf-iOCRo+gJZy1~8TTEi_3qFB zTMZtkxNQy&{53p-9#`{wR_#q4I5C8MgRPiU>}|Ld=Uir^c}8|nXieQ+5BGlgy*=)O zCQ?NR87CBtdMDKrr)jD|dNs?=CO?gS7R}6@vf*;Dwk4~!3QyUffWhsW77_AibK1=A zl!$oc=H)Gc6+}h;_9SbRcHG1=e+xQEhKT2b!!qkZ;{=y=He2ui7S1rSn_dqZzZwqm=-#Z))6rV#|dUdUpz|S z-5yvmR*UZ;dXyjh+_G7dw@Alu!#>nIO9432I1o{N)DS6M+ONc^$xYd11E&4~dR6Tl zbaIG_9)zo-r)EW|Mo&Fff-$n@Bi@c-oU7Gn#3Z)5FGW!k zL7k#?YVJt}Ma_kqdU~r+m!0DZ#io>=JT~f&c^p07dON-O3Dnj#B~Hg?(1|mh%Q@Mc zlYDxzEWwHJ{Kijt~OY-Lizt>9sVywQqpy`D;xcp9y}` z3;H6A5kxhMI&DCjCZi!l0i`QEP7>3_*a()CZ$dVepL*?f6FhAVt=+JcTT%``F~_i3 zxl`5{$~$Iq_IUeN!Uq`Iy0%a&d2C^=OiQ`cmY36G499#un~uVv;Y!)HJOD-cw#ax$PRk(2R#osr$BD2UDP9Y3PITVhW&!e*p=3(cM`5hJ z{l=_123A?2Lj7g(8f+Cd2AVMgE7UAuJo8nVT+mGB?&Ca41@5^7C;b=~-PnD0;;kLn zjg!@xGYO4z7kuyIstYbkh2eX^piaFz!`fP_Jj2xB^FdbXto;>h%YJ^YIp(WhPNAF&~&&>N;43SgbyW2-b7rJIU)ru0|3)FO$^Gs~rZT9k zvVi_ru($>^gs1HA2)2_iw2J1uT1sB?7uZDYx|fwua)N{S5nRDF3x}$4J$KumB7;g8E4-XP3F;+ znDbYA;}I7Zj@vg=CqH|#_V3C{H*mnTau+SHdxps*(_a1a4=nsao3O;1k>%v*RuN1x zX!J_AnBRNnCy>P3tN2#>R<-SG!@J`Cxc|(!=+BXKN!7i}YJOvor$Hnti)kK*x?v`Z zk{5y%mEvJD%;)mU=sf!Ps}Bhb{lgA7`mK(Oui7vwU!^t}WEOPd__fgnt1et{kb|E3Y#sy}#z!o$l>ig6VDuG(Rk_&{ zfkJ~CJ{rYNcI206r9;B9eTr`$HIa4t7#MSoZ9+XZ>E@r^E(YNRIXfY1PWO^l4>8``!F3MGgq(8 z>QfgXy*c5{RN`+W*ED4?w<1yt@~^rM!1cc$$(9-^%pF`V)EPnu4Iy%S-5m48T+t9i z>*H6;Ca9O@K3=)R&n5@;E0UWZCe$#x=c04b(j2q@4B2N>ptI6X9LX*4c4o=~3oxR= zY?k-IdBruEcJByR@P!-~i?TX?i4+6tS$K$6BB<2GS~M3jif7etiHdmeKKLz2mr~pI z604!3H}uG2=9S~B?O?+v=*Cf+y{}>FnQ{tb;@)8*AP{2JbuHiM-jz3~t64c!S-L}{ zL=gcdCy&Zj&vQE`pmgtlw&{6doZwR$N$FuT^iB~`j&=lkb0mvS4LU8jtRU3LC+a31 z>?|lNl2UnYH-1|tu)CWG6o%GK@2%8g`t)1;_-wDC#;-_jaI)Lk+Ps3JzT6DMT8i#U zVxit+yNE%=FPkqV|I(&2@N@x42s$N6g@|+hw#WpJCPL-_W4Zn^{e}5%vhaO#q4C9T z(JCv}5-bJ{)=h_l>O~|JOk6&9eW$bxqN)N`N90DfWOwJfA9r2`FUpPR{A?G>61$1e z2Q>*-kyfSl;sTT*hH8#*pOuNzHl+Y9$L2fKc3fdz{Fny% zXyez4=7pYzzKLMVXI_On^YYw-s0o(AcAu2Bq3zJmI1&JMaY(4k)?(h3_C=cML}qqc zr@f1CUpj3(gVNGAYxjJ8&c$kB7F_^r%ja%}=Tw}6Ze_e%bDg%~C}tCY6-pF36m$OU zXD(;4SdceOL{%@Mu@}=GiLfdGjP*TZ26kVRzv`hCT*IK&&t#kXlNn;D0{YtsyDG=? zEz7-Kcd^YldRjSsZ_RBko}k5grfdW1ONt-G(+~=+GY~&L9GUO_QS7UkE3|*TOK@>= zJ7BM~v!Q|p#AOHcwvxK;`*lsCfdw>#D* zvE6yOEvKVIp>SV!{iVgVV0>R9P$bUKiJ#ZvP7!C_(mX*kC2`uGwAYH6631w3gsq1J zg(!vd(9IREOi|7ehM}bSp0Wmn$aO!}Ep!(|iDlR$$N%c7>6o7CfBr`Bwsc?61t;7q z_L-MwNL{Y$nw=|V)j&eI$qcu|u)uGvI$fuKZZ-X}D@%WA83cL+8~@_!Ev!B=gLjpU zFgY8|Bw?13+SXFomS+p2^ij&%K$3svBjR|UuiJcDzPneFE@NrC1ABEEbTP^PWQ5gQ z$D3bM@MXv5~C5!^2yhzcVzI!2B?Q9yQ4zDYJHN(-up8chzl{7?T zklV+`+oI@!XTkq0H+w$I6`k}xIB7t?Yr(8D6lKfyhRv7X$0*JK?=-0wHD ziq~xak@8f1uR|2ITG4-|6oc6oQhdD_d?g&X=$@PFk6oZ>SGx8x4k~DNkPH<4ZdKbV zyJhsN#`b2_L7VmeUNF9y`n>Piv7A+CHfc)Y((7@2zAe60;F(0b-OS_@$n#KNTEp1c&9SApw&)Cr_HDn$Fr z7$ihyZsYv)Du(sy+)z>_Gr8xdI#$fgV?VS6praG7<}Jg8lD4NuM4oOEWLI&Js}i1j zwRAP+IF{61L56k2e+AZPz6gxjtSyogpgw+E7(ZGtp zYn_J)SDw!f5rk)Eti#@?Dx1KoBgtfTSv6bzy!)j$vq2b-vB&3g}3?g=8M~ zFMGUG!K08tpE(rjs=~1mekXmi1!8vMqiu2j{TS~&wP{{teroR1Gvg0sY9sg8jOWCF zO`1BVXR=Vv83!@zz8>ycP#(*$?&PXSY|Diq+nJt4cO1Gifa1-U)sz#sAutAlD19>WT8e4lkms15VCk%iB|eSMyk(eSxREu&Ko~fI`cQkEQ9?ddX*SrT z*s8Izdp>P=Cb^SohhJi%J2V~J4AGn+dNBHUPS#Cl@E~x3vCa+sD|P%dXN@pH1Bg0S z0Diwb+rCYI;*5tA{PhCZdnj`?u7Ncf#d!siLiud~ku*tXrVEF4>n5z^N#t|j{K19& zRB1=7kDn;kH8P4DJkDAx3gF#G*fVs;OuV7YvDpZ#CIpUp(efFu!4Cr?PBLF-LJ}Yf zw}fP(uQ;*im-HAk+Lz;S9~_J7FwAR0l_aAOz&kC#FD(}Zky8l%WWRMyPZh z>ZJ4%WHMV%m0?Mo_)Z9Z214Nl=PrzzB=*+2ZcVYwxbCo`b(~V3%adw?PX4B7tb-cS zcUNLOch^%AgCVDTOIOytA4wB@k;D}Y>%_hWNi=yEH8t5m4f*k41bJ39{H;Wx2`Pta z;&nXYZKpr!*VWTep{E`qU&v;YpL&DvEu}`HDKD+&JYLk4vw>qf8=5C$x)C9h=;lh6 z-fErf)vI9h=A;(tXWv{n9WFw=>9qJ~_4T5Biw}Wk7ujAFM|^SRsG%0kN9d2|JpKGv z_Vibd*6Hsn(V6FrD*sHWo&X3fmp=_7BgfHdEyBr^gGUvYIT#qQBcJ119;s3&aA)u~ zeTqHa889twYdJhrQKa<>Ng758tP;PR@9*kE`P_Z!Jb ze`XQ?m*!4;FMVv{Kdtz0|9^(zw-e(LohQJd)F$O9CHq#fWBBNL$gotb4qe$%KY8}bO^-xa^lHslLi3UVyR6**wH1)`~ zT8tq*>ZQkC^wW(?hF(G0n7EdQb;$%@X+s(@8It|>Z@jN4-M8h&HVd;qA6Y9Om^h6R zf1ao~USClg^YFC*^~-Ph-hVdMijY2MF}UmDOsk2dys&p}wl%v4eafjsQ^N`>+YHx# zoj>Z0hPZRQr|tQTN^}g@()PeA<3>75LDxN&9ri6x5#C$k4MX zsuxqQ%u6ar>z8eBT~M~&lqy%DnJ;e~+XO;~pL>tQZ6aNjhWo#6nEXV=$Y-Pd&Lc6S zg*eSxfdGD@!o=iGP7V=0n5*a zi5~%rdD%pP%?E)kH)e*Nn_dNa+SZeuoizlpT@ZTgjgH*|tq^PA!Ta1}FlrA!S&j}S zcNHKZIZ0d}l*Rh^Q*Y?CJk9Vgc^WbKb_5fPU#2r1P&PiG_eg3CsXVX6QS@bTX>{-K=M(=jX~$nyY$XzEnE~31V5WoQu#%9 z>nK+T=*B_pmdD=B$`&`{wx5al@_?G`FOfSNVzuV38T2{XxK+V;qfZe%z2dgc{Ugmx zY~SNloNPqKd6GN6M;h|UWW%4tEyj2L&Ug%EpM2{7Ge-aLgC3P=X=h1rkW73mT;EkU zZ+hcoPpk{bX&|gG^tHz8Eyc$Qd>i0;@7xckO6#|dw#k-%?cOlc{y_$Y$ed`uftU7g9hdf!H4`XpJl%wie{H=MvxcwnhV#t!(H`-hv7@Dvc7;jwd z?I{OsG_7&h9zNQWxRngEZBlz;bL?8kKrMRTwLW6fhOIz>EzH%@lUsda;S!TrfZb)5 z+vLi*cbmhqS&7ZeD=3ZsE_>kVzp?<43Y;PmZQ4H0)RJzX@O8zP2=4njIhSS6;59-e zT{>Tqi+vtw>%k7*O4tA%ZCt2t4J%QdR%JCh8J?9!y01rE-h0D4nArkDPfgx=Fp5_y zW6HaxsZ>C@Zzifv1}oTZ)!^UJRl1JiONwDA;AEfEuTShZvb&pF)z$ntdCEGcQ51#m zce6T1$}#&an5eD&(e>usKYg?RL{s^@+J2p23&y0|F>C%BXIPk>q3iSSs(E1G-sVuBaHfvS+v8%>;8!L{%=K{f4_l$Je%NJ=NG5?e-{-N=CYTR zS3cQ+*;IL7hw9E^*dtl&<;Vn}{o+(HXX6MwKT%15s3gm49n95dz20rr&~v0&_ODFJ z*DSDkRH8T&YuUsaGtT|o>NUN>e=+^Q-n8zrZiW$r1ail zr0=Zl9`+bjfXWTbej!FJC;JKcYrSvBO07yD@;yg;9nTVUp7mHCU!Ek&vTV_6Fxm6R zQGZ?i;IIazd(_QhF6U#+g0AEtO+wZDz{<1!f!;NTJeD+9;_fuBuXgG{768>#?DgtA zHwSL@H|xgkQ~2(a^Ftf!yF+JSb73V$)>ZYLY*M8gpGvQlA*LJe=$yza_ob5hJmYyGHO$go~3@m;gR0cXQSko`eEt$s;#f*7cd_=HQu-Y-WDcME&v3LJwE>zQMws zfc3Vo7p*O4E*;Nq8EtHA=6ILQteOItR5zAk5y_V@j1&t$L##D$5TAr6?^SacG;q!b zbdx&m3|l740r^760Vf?fDk%~wEqrHaqw`rpUIV`U;R8tWfk**~Aaank8jG>oJB&6Ck& zoGzVY+}2rB@Xq=u7JdZaTr+G&UFltGKfQQnpVeDVhRlTtt}TbLhFhVD7okvB%)On; zbVKzSA$FHS-PiHMvr7lZV5^`9dZzyfeSSLID6eG*?zGAQRQm%$Zk)gLri&}SbK6su z^8JgyLZ6)RRp}V#in(4rhl0SG8grj1fsLjZM;LSrqVBy=^2w?j)3zKw2r9S8VljZ+ zKic>(A8Lp?z8I_i%(!vX5=e2VZ@8H6O>G^I|L^GL@wD4})z=L-Q&svhyN5;)Ga-=n z8AM8gOfzCCd!S(a7Q~U9pF9Hk)*vOM0Lfb)8OON?@7blK5~dzteeWAM@G5(ip?q z4hv-6Qn%tiJ+oc+2JeDCQ-rZG^DfUMcNj9s4wzu38X?i)49}#ArQ2QDcZe_xo-yW8 zTQ1OSL~MOu#)a&ufeEKAK6dy+EYe403-22+dn?&+#22B<;kY`lbgRsNZo3X;b`rkU z`&0jc@esJOtcUhKwns6{oyI^CJr@ffW;NKE6#tG0Y!rp3 zVj8QwMQ?`~xWS*!{(4!u?_0>rVX6YmCP}cW=r8UizAy_vW*tmVUAvhK&T2ZDjE1`_ z*-=kDFPW0WE~7ZmccRESokbS)@z#|df1j^m#gh!n$|*=W>45O-;lTWGtNa7g)*6Zw zCU!$>UUT76PhPLUA{!UTnJ5S`4>_4G)Y4bVU%f*`CL*QF(MrOE&8xcjQ<2f|Ccb=& zOFkr;o*Rv&GR|k2iZPA+xJ4w{}INDvR9`^X4A#{Vn zAKhljrV?KdI7brobcUMkW%O(mE0n^He$6{KMMH&M2iGxC5B|9+L`5^B?*q9N&N|aE zg;~Bp#xTJ7dm>w@l3IG^adR;ycu+8SR>%rpc8qq6Gbb{6i0P!a`-aL!uN7x-NPg@& z!_DX=TFY$O=zM~742#DEXqce(DfQf1pS>*qp?18_Uycw?ZoJb1@uUY%ggP=R<{M&E zjwVcZ(BnvE!Zs(6siROJIJg|D6O(eDV*?!@!JiI@9cA3niUwO09BR#!84-rRX==o? zG;E0_w(OP2W8YbP5lnsY@q~W_+4tkt+L44$5dlK!;uTNQl-=m0g$$BJ%Z4m-vBp_htchM`2ILD0B_5e64}KUb@ko;lK2njfYV6H#>F?72i6SB zW@*gGFQ_b6_q9EHgqOXN%EwyJT+GR(4l|ki7_LD5qt_(}CB4uyk(DfT62#LOgi1et z`FJx>=eZV^O)5dw0)l4EVR%O>F!^v2rQ6(mwZRrPy+<1@jXwcFN)9kyY@~dfP|<8Q zjA3;DE0NP2aJjodY%#ZFcdp}EYl5-;0&&MS(#_0aai{P6GJO*WXR>(#vYSC>^K69C;+`huYd)=n# zeL{E90#dTDtL#d$`N-$KTv`Ieb8cPhTt=Z%WN;;(?NG5Nj&o%uo8Uc57u13HGQ+FD z@&h9{sz|KMb@Oa_-O7T0ttLf7FlOc4c+{9@H$r!BMMW#IR1KwoC4zNB^|bQLSIL4X9i}#Xt}dP>>)|iIR~h zIn!j2C^@Glg9#9jD6v7H2~CpNG)NYj+~h_w2(-}ToEqNY-a9kD`(|pU>eYKyum3%# zy3YCP>~F8V_gd>Sz8RX&ZVD?9)1~CwCPaM#Mfj51rJlfx*o58M#YmriY9HlCyA0bi z147xIxg%w05!u@0WVvFcC}JlZ9@vv%Yi^qCG1tkR#yBj{#N!j~=av%(HP$%&G3@>= z;9m_c;%VL$mA(TXoS8k67b~lJ>vyiX9hN#3kQ`m0;?S?MGVi6IUoqRuK0PzVIww?3 zT0V--N}CDAr<{t{Gy62EK9_C<#J305@@d;`o4h1c@@tazmn{$}pao(J_v~#p7(5(M z`#mQ#_3JtbGvkMR?Y;u?E5!>oh7e{6-$PzkPBG0x!`<%4G44gGk`Vufv0%W5q3GUm z%}o4(k?AUuFSJD-{lK)Fdy(e2ZYJr&Ra<|4-Z$_`fA%mrf5NwyM4C<+JOpyD$2m|g zyL``Mf4Ff$o&x#=!2Sn6k1#t?B(2$@&XzLHTPKxOc<3Dn*JPc?zOoEWZ(F|om%oL( zM9rZSpig9CV7jGuqG&!Z04E@rD0F&yr>cZqLVXjiLw)KBLd-verVd1CA82KMn^KP3 zy6w09?yMKD8CaoES!EIh;}Q=1U9lmRQ9N{AQR$4^kIBoTyHlbI0oGslPBa{qEGvYk(bgurr(?`3%cVVe5?l#tt?xwQ(4l+)qt#-WSROg*NG&72AYlSyLCsYJBEf=n}Y{|Xww z*y)W+?nrG}q@S6i#v1XpJM*>a8J(;4t_}ph-K5{oSbo(4Z^*OV|MVxe$dHAuO!6$J zGO<(_yQ=MJrr3ac6tyl%Z&OKnV!*!^0;nUh`zWMtpxrn`W<)aN%3$8mF|(FL@c7rf z{{1wYL!Eh)c-@y{$c}m9u(EwE0TL)1?*P&_${wrrd?Ms%HTS9ccJr5=@WM*jG@7vW zJU6G-X}wsKxPqOgB3$>ULZ5(+ANRL_e`sbO$jxtr&$WI(BW!YIb}JQ`sM}{E=q2N0 zlTwyP_6yox!-onsJW?5&$iLLtJEDVRHdbHMS!6zGxNRu+w^V^9k!S46&9wnwfF%Fi zf{a8V46hL%O&0~1f5(MPw9s1az?&VUEgJG`&-vwdzm}^y+5G2}=lFqI{7gBW-))C9 zXYj)wJxJXNvUr)4{L|HHf;ByL=~3Q0oXN^BXT9x!sm()cW(LbPAOxcaioJ}ZW)bk4 z{{~D>-+>}%Bl}zX%XfnN@w$tuq@70k56AEQ!G+2mgCtn-_Oms(k#!Ey3uC z%1x^&l1hO${tLuGjRB&*S%i4Wzk%@xB0#?$GOFjL zgbuT_W*XwBjvbf5?%so|4ddjiIb*x#JxtSmIfY4wrm83Reln@#H9pF`YP)cnV)$0L z#K%_()zE*eufI%zwP`?`|0C9rYSCm4_x=pF-4v$AJ^#YHrdvW|YnYNqPxq&#zGd78 z1y8wu?G~jb%A8S1?TMd_%1coN(aQ8r#*2;w?y<4Xo8=6BPw@Wgc4W)oj!eZ*DfCRc z%&%mTxemJKi%Zyce_uE&-q*kJw^>aQc8(ozz(_FuxoYPq*$-?Ue`;3SE;Xxbb*y@@ zp}XGWw=d#oITAfL`_Z|3ugKVO=~o zbsWnW^2gqv0`Hj8j%8^m*Js~S)+`A!7N=?;Uzx5;0g1?f+FgZ2Vn;E!@$6m*gS#`2 zJ`%9LFuq>Nppv2M&WE6KSL)7Q{s{!$PNGd=Hj1PgJOuxgYrEnPkE=3MFaQ**)oW+P zg@B;x2R_&A)*3s|X?`@pCp0TlNTktV{~^_vR{))-nbe_O?IIK@vGzfc)y@);%_`mIsosZTDEuToU# zw6v9$g0-_(LaU|nls)dV(Ww7tPQHL>;EsS61WZ>?jR!x9{}s0zU)s9@ zJLU+bzvCWhiKhJuWYNmziRYh*FB@pGFZ&G9GF3nkfhTIQgBH$G0Ztd|uYvX(v9+#N zM*XZd%?mc4h$^E{mRn>5(+x|)HAoz8*k=0v+yke^C;!pvD=i?W~l!@R@J6z zIW+;p8q~7|Plz<UUYYeUPKioV_KlM&IUrb^OOA(bMadF$nLswaDtsSdhL^cg2rG z&!Q?@{Okz-Wiumg>TTYU2+nWECRXNr$4R%r(i&{=OoGs552$DXZ5-ZI_DMSgMGTv| zz5M`Shl*?QzuNDy&9xzpz`OCZXr1g#kY;DYVU5=bt$-fq$}Jc?w1APS_t9IH2#V~rou#&A?nr9+z>CU%oXTTQ z$VWXZwJ9p=K2(z73&JRtbSB{6N4?g@LoM{U%R(kFWlLcbZt8YQB0e|FQRiRreS2cN zF0DcVpG}Ke{fEzQtJoaS@Ow=R;;)c3IOZyo?xtqsBKl$hDXCil|A`86sKIzVMmXgU z3D*Y_bIo@oGgH++KUdQCj3)}e^{g(aYBSuYrEP=x3M^JJzb5h08!jcW+0FDZQ{qH% zaStUmb>Xjy4QUiCdfrMzIxxq138$$>>nHudL^^N~@|bMDh2wrvuz>ozq0QqMccGPo z?3Zf34uPDevlkvoa_io=6?4Mu%lsIy@%TNpUK|VCEH@20&$oKiS0ZMKpGt-pvC3H1 zX|?L*2M#RjN5@cdco+lo13z51E;yO!fc*csoXGz7asui3|Dv2w0m=!S?mx?kp8UVd ziTp;RU8ORD4>2!gk_Df+Y8q@Sepn(#`GFZ|^J8T{kK&}AaZFRnPWVO^bJ=J-%MR=? z{uRme-b5s~fg`GCR~g#HfX?+lmTGi_OBRF{KRuk2=tjVGecbqet^#vts6=#Ql9zkn zsZgSpLaCaZirlaBf_>F<%l7Lu?M*;K6KF)${2+tYtyBx#|-u z=efwD(Tm8F=EervBEBAX_RNA+#8e9N!8--w=(7qT9>-nwFf1^JU|ab~QLQE_Mm$OU zFV=6ONR8T610O3i*)_M3T}7o@EH{NjLD%$$m5&oYmZuWvoW6Lno%VenTJMV@glYjB z8~#S!3g>txa~dSH}y16k;1Xg zI5AuFJ`AeFc;A1ocpG*m)fiibbM)2w+yUkd%d3O9;{HFVC7Pv?T>HzHd9!}7BS4Zd=y4LiX`fFT)@#4+%=0_yK!G& z<&2JsR;WGK>jw#E$A2G<5ty5c^;6ZP%7 z)XUzGm#b!Tp%Nn!>~0F#R5Lt$WRL|a{v-@ucx4QEUi6ea?aS&P@^*aEuEBo{kVB`O zID~i`tn&>;81f7U&K#X~F*AfHd1Lkw46ZleoWHKim2`GT_>Wozk7K#N)=EW#hb{vx)5e>#f);Ya)c?NSnHcVrrrd zfkj73rzuT+j-~bkg$gUnQbxOaYO8HYqM4?fI@90>+_L%T`BZz)kVjB1SHY*!8S)u| zTNhK*XMk5%x_vfqh8oHgI(XZ z!3R<5pvz+ZOwicuPBa_pbm80WtQQ>lsJOEXaRfmXbu#aL;tX~& z=_U%wwGEEy_oOz@wKS7F(a{?+&WXOMY_G@ooNjX6fOBRP_iMwZyWuIyX#{R>L06>|-OM5OzqrQt zmvj*VIBXB`0C~)k_HsQZ3f(hNk(76uwUJ@(Xp`&XV3Lu$nmBE3|8P9J;pRZ#uS@&R zH{WNEo_}iX3wLF0|1df6t}~_43_D^@3EGm@h5hpIfX_1Aq~mS$@^KdwXAayPw)Otj z=MiJB#kIfGKPSm*Y_dp5OkD^qdPCRUXY3Xlz2PK4x!BCLZW}BfLEZK((c&|`&dO?R zEyTmNmoLdPqv#2U4@yh_$3^kW_G(#tk zmG1qB)NeLZ;@>tL-Cy!dQ7L=P#IQz51N3sENeayn8wHHuXs3at$I;eegoy>r4ozQM zwQ|?*LSHCXNtR1*W3IA4Pu>uC{%`a zcsM=Pixjk~_tY)v)l_;9S<3vVBG-5Kt!Aud$zyEs8Ii+7PIZ|f`=a+rHbsqbUJG1I zd zKcvO1W4qg2s_g>`d*EKEL*!kget4d@I8yiyIMd#T5pbLAgb9@~4$EzOC?FN6#R_q> z^16QvPK_ahil914k(G;L{E;?P^E@DjD+H-h<;PM1o2^mcxcCMdru4@Cc8h6-hA-L;6*(eEYE^^pAOS09qN-UR3%L%;2b+R{ zy^o_>Gai^m8lEhP{S4DY3w=r#nD?INs!2aH{QXqH$|{@H=A<=KONn9z@B>JqO-BfW z(J(G)BvMIrU@*ZW%moCxlYV(dz;u*w1s8JMo#us^--FApqPD|Qvb0WODcv2oyg!%n z@D?%R{#TTv;TKn{t9jAnibD^s^%TFddFT4f`falKW`|@eqgv-eT;ZVsT*{VjO4c_ zD|{axlzB2kUk0NS_Hu)(GM$OKPjpNLJYCH}Ypq=A=In*Wk5#2rjXc00ZzI+HDTfk) z^%eXqHf~KJd4+JVP;+*bi9%}V-nQr+#QKLhJ{gaKWA?Tb)PBiEZ)n;0!`aP<-%u39 z*(<%V2fJkOCmwp4ZxYyZ|77uR58!vw+e(U(kAe>s({0WQDM{M{3CzZIi)<0V{Q~?>|@Vn>Mt%MlN)M`caQ89gFo~bNBNpt)*fO#vhE9vIn1$)cldj5I{Rj zrUt%7O2rmAqHBu4%393^Ll4erFYrQpNkTh#r)&*Z;5IJ zXzfzH`~5$VF|xyzb$kvi*z3{VqsS_)_Ft|UU%tPnoXxXn9X3QH&o{noFr#c z#E(8#i>I>^p}t=4z}6!Jj``u4onQWY_qR|&^)_CR>fG162);U{>sczR%t@_)?cOFM z6?3X*&@j?(p@>mDCyQKhHB!5}6MeH9KyTbm((4H4S-F{uouwWa`n9wpaQ?;}zztgO zKke@Sg6%*c%0B>uNsin9a%0{Wxi{c4`GfH?D{KL)^o^sj#AtVXn_AwK1dyANEa5Dz$6T)8;f zS&Do}OS@ht4p4120HLlDz2E79<078hw9)7!JtpY9r?CcDBEa77FP!{et>vE&`_3d< zf#lu$y*6*f9QDs~L0#^kxo}2Fgw;r_qST{(nyVC+m1&+nFjjUChrU?1(?o4)GVLXu zdJfg7+gF2G*N9W^35(hf-9s0tZhQ_U;a>3_D(R5$j?#o_*mZp??RKQ3qT>`GUQm=( ztbU*FXZ%x-A7n;0NZ1!{hq=d{oG=Q`eh|Dc-U>&$!%l z+&4-m5}+>7#Q8l%n099!JG5irlt%Woiy9OrN`_vHE82=f!t3wC_$w>+`1@0*5XSqd zl!lp+Sh1)=cF>~XltHb49;r8}hhPD=a11s{IhvIKLGNA&AZjJumR?%DKF+O5sW?Hv zjrD;{d9=lc6{}a57p`dOcm`Ud1+uyM?mLn!3_C&`Gw3|BOsI-@)fV_^-3aXV&@^f9D5_+oN9D~q5Cm!R;NY@`amRQzmyUtw3VA;^>XQ+!fYc2wRmkF za1x6_7BZ(y@MJ#4-_gynjo20Z!as4t67<63${Xk&&H1A&2seXh<$5b8T_*g(kGl|J zCGYK)1NTgYH&KD0@BadqB)2^6FTo{*yi`1I$hWG@Wn0g~k~1U8d^tdvSu{s}`oy@l z^5x6oao{nA!!!^GrP+VtoMF$$;0?WxX1o)y?n|yLaIq< zt&$#glmy3K?Rtcat5sjLP+8;+L(JWvlh~=1WgC3@Qx4GWX3+tbzD&|p4##~SV)EwK zW^}AP0``Tv!gq|3%c;e$CYgXOiH=kKQ(2SZqM~e?aU>O}#^N5QI`Cb(Z2bnfY>!y} zQ;`2xQFIv`|5tB3CLI{u%+>T^eAiX7BSP(%nSU(X~+rYE_;UjuxH(|{g>Bxoa+}Sd_IjmwVKJ7z^Eb#L^jm$vs9tg>Pmqw z?09nOL8}!QG_IC%li@+ljN;0C{(^7W=(^Nu+ia(MWV3#_U4Ti7{d-fT)6s=}y?g=c zM|-5U9y88N16YbQqW!=EAcm4EjxhXvv&&v7VYMe zRLku0YFeG`S&tjTKG`~(R9dNSI-qx42{ZTUUb9Cwo8{HR3%I@!&f-E_(}Pl@#x16#L$;V>Fx2!gU09g>zi;BT#Pe^Gx^p z+$q4HE|Lz7IP1rC{ur2+9}+dq7s4LGXUUW zIBtX?ul!T4JHJUN>t1b;MfgilXJ#wF1?xS#+^;(a2M0@Wm1z_)9g)w|HwIftLBV_C zLFef!$KvP5+^0V;Jq=5j_#}yp+b74Pn5;t}AW-&V*=2R~X%M(5srs)O5(u8hK{R$T z!aJoSaVz3nOEqV>tfxQ8oF8-r=_u};Hg8QjL!f8(c7gQpG}k`lfM#b`>NUg^~-*Rh{Skmh*(a2kk^+3uuCTh_JOvgeOIOFhRsnYPh??2y96qQ19 zo8Bmu3^xlwH0f({$&WZZguYrpJYDVP2Nd0vvqpaMggw?-fwc6s*kFewg7Z#;!S!EJ zqs|nziMpKj-`msnq?TS(yL^1oB(buZo>$!ETAo>W(gN0ad1%B1Yp@i_s>+n5ya)tY zDx--;5r9*&vj+M!@#A4njfl7RbbV6AA+g}XQ`f1QLnY5f)BU(;{rRb1?$>a;<6K2e z_qz3~O>pX)UtUccZtj)lH@+1s{D5A7V}t=xk(S2XJF=7@lT-_a{>)W=4hgif_OX6K z;xfLx#Qn$faLZZt!DkVNyaKv!L{(*<5))^D40;Jm~?KQQHD&cR7z zVioDnDDCU9Pk&-y{`#xzs(|&OO1(A;BhXZIwO*HIp>&(5xq;49tr2!S=?t5OD+T&@T z+_-_ax4qdj6g|OU)0#kkyg1rz@Ok0#fk%e#N@z__(EcdHrtegCF3*w$7q?&gjUqcK z+R%=sY`mhlQ(R21M#U=M_wu(v4qfptZ(^JwCNQR7L;n4FqxH_OkKE1r>fl98Mx%cf zrwwO-#%z!qWbJ)fldMs_vSz?&QpHBxLkY~RzhS^}ll?U>7Ig0zPr|Xqz3(Q}v z^g$H~q90%3As0AK?H_U{V_5Y@`_V-S&az3>K#-`Zh2Q;rFLvxO4qRHrU z^V~ApZ(CtV_E*?Mvw5nrDe)XB4+m1}CtQ8x-bDZpynZoD$yQOmaU9~+&f}kW!PMUB`hwt@|)&1)G_i?kSDGvjQA4&3gL}U z>UNJ+!k^?%C)-AAa8_dBFuhF}T|q^$mF*ScyGSZGEt0XH*&Je>@`4Y6ig`Zy=)71R zn_KZD63y3V6RQKP2PI9G{BpJ?IhgLl$+n~8IP$jzRZl2HgWz4P` zlyfv0>Jq1l1kiWH{DnZe-nOo`@PYMScZZ9|V;cRm#*RlMJ;GuCWH6IgMewBm_7 z+1tflyG=Fo42z!bRo*BLpN-uKOMTGTU5TjnUGb4NEg5Dl@lY+pGh0T_&9I>0!w(h7 zwofp3+75y!e1jw-Y&qg>*)U5rR66!e{kw>r9b2~R!+Ok1oXS{*9%bodW!q=o$t$Ew zA9o8W<7Z5`%)8rsAkuGy!-D8WWp)_s^Gv|L3PS9(ya6}wXgwVgm+*l8w&f_dipoUz z*L9I>s%d82>)FvgQ65%F3@^keS`mD#2)E@3CxELE+4~#bNR@mO9o8Yr0;!nwnOR;G zxIQ!Y%auU*)_ihv8rgtpk`A@&EPvW9b!Q%WjXcCS_XNyzCIrQFQtKkr!aY42t^e(b zB)-{HT;BC0%nYLRJk#}S@kb18ZQo4j;voyN-?m5bT$#%BbiogLQ2FX`len2D?GK=V zUEny@+7L@$8y3}v`@~hrBwq|9AL8)K3a>N#Y`w$$#?m1y^vvV}OX+RV_41Sk&>8Gt zE)!J8{|V4jCffVi Date: Sat, 1 Jul 2023 23:28:27 +0800 Subject: [PATCH 031/337] Delete time.png --- picture/time.png | Bin 31405 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 picture/time.png diff --git a/picture/time.png b/picture/time.png deleted file mode 100644 index 59c35a32b241d2f387a06afcf311e54ef697ce94..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 31405 zcmd43WmH>HyDc1`KwAnFZ}9>JTHGCqySta-?yjX!+=>;aIK?fv6qiD94K6`L(Evei z=zHwmBj5RP$9K=Ku(DUi-Wh{6=kv^G&P1pvNn<`IehvTtFlA*V)Bpga1^@tg;wc*9 z9Wb5H3jlx;AS)rN;gxyVuI#WhczN%Co|n1?A4QWuakfRpC&o0R_5PG=Y_iWH_zBM>qk2BcpWaEvuJ+GO$c-r97e6@EqTef>$Q4K=XJdEuev8yB zjx-Z0R}bq;>3NB`XD;Aj^#>;655Q04Jj6qMcui@8SR^6+Us_->*p#tM+uzAnd^`yF zp}D6JFkZm0{cFid^-BeA*)wOpHp~Mi^uyGdf{}p+0*7_)`d3mNjx+wjWNfAOPrI_i z!xuljA_#LL38_wTk_DpPKs5}W?P83#z_KQgS#k0Tz#k*rWc9?fCS7_dVP#GrD zJf6r9g^xPn*LU93^hXrL^N!ZjN(*%!9g`}p#S1?oLAgRyE>dAYWR0PX&mhj_f{`y- z#{<}lFF8M>flAwp7&ZxB?*!Rde(3)(inKq)M|VCT5X}7qy{MqX`%HUz2wP847|KhO zqU?B+Qj|ZJY)JEYwEMn7=gQT(B8iXd#5Z;1*`Pr4{4Q)gC4ho0qu4PUrh%qH zcXY0`*bou7jfkey>eK_b)J!D>N;JbYc+K3Kp{#K?Xkl8f&+`Q4@9~btKFF7~u$2&L z6HkuGQIeq@^5v%&IYJW0`|gRL9M}1l#SpaNR(I+hLYl6vo2jUy2axi<2>Fhqc(pgGF%-aXcvsg&7U52=niEx zb!{;sku#oO*=Z|IKAgf?5Md(3Z(AlE7NO$wB4Y|=zezh0LT3TQ>I9fc6=TxrV zx;J)u%Rph-vL$eOhki9rgGTxZpSE?azNR6K(@|rxbv>b4(o}J4V)^AeJwYcpO?bA) zuT{O8OSVr(92=LqT1&#Hht@fs1~WKC739a)`cY7d#?G@fB(>g(`8a8f3qeO@3M?2b zhWp6}Rbok#3+Vp@!>;>$H14wEF80hci08Gvqp5pon2MP&Q3oM`3gJJ%(iQ zQc}3@@3jKftyyXKM{AGQtPFaj*>P({Y@AjoU~8u?c8V z>1}E>S%#Vj+fQ>Spfr? zj0W~Cq`Q?9Gtxvp`Dsoor9!7m(y?9oPVipH`K81g-;iB>w?Kbst-gX5rP)fk>~mWQFA z^x7t;N6N~t4CLO}3_Zy1^tMbqw&$*h--8^N}_Rtl`zksQ? z&cdjb&VE9guWV#iu|qM?iovqaWpif1`)~Y1G_1ddz{*+bYEZ zqRv()aYctzt{@|5f}Qv<;0b~lh9>~3iQvrE%E{vvMI-^rRUuX^AVXu6RBDSmej9{NjV2L{; z8a%`UuK)UM*}pZBDfaB7<`wyBjb?;tdq&*l-dyXK@8}o>w(DMfnIoDf`H%Uh_vs-@4AXV1BPYq~6yBZE$i%{r7iH0xR=4ScM$z4CDt-3VMS zCJKaCMrhf}WP5(M!z9LnNm#!|n=HA%rb}6_B&>ba}$PC#^Lw(YR>zNZhDpJe;o)+c`}(`Fx7*HR%w~m%P2x z!C|@*-$~lVd=0i6ABxW3F*mK-&P$b@PD`xclxp8yEspB1#7#*o39ajB%mh>l`D__v zlVos17S(%;e(q})WJPUhN|O8x@c-OilLc1|6z7i16*QV6$kCiVgDp>UrB-nsiZlnl z)~<4$q@5yfcVGVA`g`DYBX!``Ak6U#$$kT<6i>Mz11v0hE&+1P`DQ8)Pm>?FPUVBH zjdA<%v`9{WawJ9zb&?*9VZ{~^7@6>>lyN3vh6YO-CCK0buO0do2kl$AgB+SX^A>vg z$=E%)O>*VnE?rEQ5-h*$1(w>Q6`s?PM*Un|Xt-9tBz42_jZ~3sA8v4^O|Jhj;B07| z8(~LhlT{}KDy`0SKbG}!bvuKhNG(aS1dn;kF=((srd!{rH(HgyZQ6N0RW#v3R`_Za zYc_WFd%c7x-u$^;1dX0VP=i%(e4MEEDf9@3Da%|!2V8x_nEH|6sLiLqmtoT{ez2jK zvURRZK>(sM)cl?oc!0r~fQBA53kAUm79n>Hu57+Phl_REe_BB{1-0WQEL1`)qSE13pI_E%`<-Nc z%N^=v?tqz-iW4+b9OUx6U7B7_5SzwA&Wj~XQp#O({&0knPVIG%;X38uHGDKCk z6-To9KYcW3#w3Sr*axgQ2J?R9^;2^mad?l#JNl_VeD*NpGArI4__Em?#Z69gAuzo) zRvH3lie^{a1>AGVZOw05mY6%J*R}2*5Y&4b=P}gd|_0EDz zcQWE6EEHR*zC9cgB-ETKlMqN1<^fvwh}Vr?pr*^5Mp9LN_uG7PEv$vlCp5c0m=1H| z^JJ{h(Sb`1a(jMLE+WHt`SRS#CZizC+m6J=KvS5($NRnNz6SV>bf{u@)~i0ro{|xd z5XRk{CgD%_W2+C)ZjZAV9sQ=Px`XZ6FTu4p60USwT5P_zVD?x^_lx%{V+#-|O?w@J z(YB7R4y%ZlgI|)3E{5Ma1mu)*1cnu6003)GtgGQT2qdSMMv+i-afP_mOtMl31#{}Hn_cBHf3w?@|LL0_FeKTlynboBtp*~zuXRT^U3?;pWJR;ZuhWk4Dz@R?jP<84I~h03%hHcq3>+P#CEz0ient3pZ#38@d?3 ztxh+Xn#U#E21mItyjD>q%eIY+E<+2AZM1f?y%NRk7XIRZeW}xnv}Ac9150@YEhQgt&MHX3`o1U4cN<%QLI2Y3aWAyKqn^tdqH zn;*YQiUf%b|CEGh8;Pl2NYUzBXvcSDr4k?Q0pCRVmnqj7KG~L8lCJh?gVfP4AEhsS z1q4Z8G1`bcZ`%7Qr%n(6=(@VI-~~RtBNWPGD|T<|N)j}j0_sQxOHOdh=3FF-&$_Ow zPS81G$)CYGTdRi%*SByFSlwczzteo$x&u91hoO!$x833`4OXR+Oe*oG@w7%K#Mb$T z_}-|#m|}k-)QO1%kX?KGL;p>-FK&J`ADa1Xeo9AH z&vK+0HpwF^?Gk`wNBgq(!<5;xjvoCcyM8vsm0s}hn+r=ujn}~`Q;6?w$Ex; zADryJ$9#~G3nrvyPHurZ5JBWVUtiMw=LbZbBn2+q&=OTu^!w*&X-IEJVH)HOFY~0N zVhAD3*2oUHHS#8|sGN%PCv@dG#fEuCj5S3>7U5k*UY4(VV2x?Ur>K}rG|il{Qh)h6 zkhl}+ZPM+xKQwiuh=vqH&znAyHHje^1|uJUDCK@3lw2?e3lSINs|q!P5q4$FT6l%mvAU*Ir_y z)yfz+{NCX=4@Y!y!YRF`R^@0*ovLJ~;4uk>@m*0vV#xnN>4 zyKN}9vF`=hHgiVe>7G_HokkoP;wNAfm6Rmm=R7gRWf^Qv7=K6fqttyBn8n?z;OUli z6t@M55-T9rB{TdQet6VGRA*U-r^IUnmYVRPofcZZaRt@X63~BB#_??%-p(!yJsmpG zA5MDq;W-I(AQ_I8idERr%4%@-C!MyeA+YU6(qEgiIZBxpUd8Ba%(`bPk|@bP^MGvf zkjw##o5T=lPgNjp)U3)2=xTvG76nAMp(Y>g5$Z=dFB1|@1RcRvy$MCj1v>+H;^rzr z3&*&t9~}qUpJMn>bcnNb1*WbXI^6lBuW))~^&XIu!juJ=&ka~}41AxfvF3m~Bxz3s zpHSv(?MmMTv%{am$V>=h8UkWm9l3?{S9}Mzuryqat==xVh!-*2Eo|Ru_3a{1XT^uY zZ6gZ3@!p!n2b|qAC$#vc!Mfh<_G#Csz5i7{RpN~nWb7KsgXAj+ZEk{y26x#I-Bw7t zsDF*(hwE>J7HGC(wn_-6T_)u^348m?M^7SSC}o?e)@8qgYv>&v6=rVCkYtdR`bm?z zHF?}+Wd_1twOMCU(Ve-D&R0XemHXw5hKa{NV_Lwscf%_5Imnmzd0xVj zH!1Nyr7G68=1EI0DzDht+#1>d@@zdUW!k@=PyFd|dPdf580#OZVO=CsW3%-X!`!Z5tzudviY6L%1XE=VHQ zE=r=(JHaVocc9>`ScU$&WwflqS52d+&9_6yM5y*KVgW~Z@#Q=9+CIm|QBFz3nR?^< zyxt_jPYs?zvxIZ=6iBH$joXu;jMfQ#ZE^!eZB={~*z>DSG&kTBPM8_?kw{Hqbm8c_ zj+(ZK0Td8W9;~W2rP5%O>uq+y#Ebnawje6~<0jIY;B9MlW<>%@*LdhRKhX+9d21pt z&YaK&)sxZf@1UAj5&jrs;?)97C(5O=yomZH_Xh%`5wHJ=LQ2SuI8$dm@Pj>+ zU40-|#?M9f8pC(6?g({JCUZY?6walg;>#vl=F#Th0L-q6@+I^3EGDFCH z8b$Y%@P`t7*;tUYQE zLuR$6vUQ%+R1wbeNFRXhTlmNbF?}!)s!Wq?o$ld#73!;`D2VY}HJqT0OEOndZTP z{+^h~8gi)g&O@vSlGtOkXeq=W4y|3VCekDIv{a4i*m>I_c`9^$R(y363zE7o*%hET z6WQo)luTFTc&qA)s(9}!ke~r;v6@ZK`mW9(?Wk8}Zmsy&ZC$ zULRq)V>;Q6mfX$M#;d7My*x2`Y7^5Sv)nggn-2tgIrhx<9vl8-ajNK**HFH_)m4Oo zeIV_(Q<%gi`bKFvyQ$I9AgLU%cVoGCC6%?BH|b=~qR^ck3&+u*sI#0y>X++c)yC?j)B-xCsxv1tX ze=y%_#_y8AYIIDZb4uvXn6my%h2XpTI4L4Oh!6uQ0a0uCsy3NoW1cJz|3(7OHD6Am zaG+M4o8um5)9;r*`{ISe6GGWoUU7HkTB>4`zso*KqLqSw3)pd0sE@nV>B9b@a1&&d zO4{(w*ASnt2yU@@KgR32iOu8nCHh>a@aNgFkQ>6@?b%_^ zL5B+$wpXH|r@ljtgIni-Q=2Wenyo?EE9U_l5M`*K{mV7g4PLJ^hlPY#UF)`0wc+!l zXAltk&Xm$c%e)V?T?$bzu3iTZ;^iZ5LvJI>sXH0kaztbO3YvlYEsIEtd`aX*SxU4l zHVBh8awKArzcYz-!@ogUZ`ox(dof`vseywo@O}#w?`j>M)P^dC`@}y)U+gQQsz^$6 zc8xb1kW!-aHsoaZKm%ZThWb`_yn3cM&EZ?S;Gqlen%7HPWSdQ_QEFF#_s%|ZlIn5q zQihNV|4?P^t{t>~Hl=z>do^7$Rwbu&++qb~lgyPd4;N@S|MMO3fyLL2X<#QHpe z(hjkmJEDogSeeX+PDE(F)p=F?T*GSy@?Li$wKc6(&y&9721v4XV0@zP{ZzGDo#c(!tqq*}eGe)@i%t80eKytF+l(#^{P*mSy z5m842Xd)NmWao(LsZ_t(^B-&xsM9FEIO=xi2q4G9B$XZCaF9i8>atXyCKSJ61%gX@ zKAD^a{E>oWj(N!8<@?KuobAR&mkpXCi5O*vpCh&Rmmns0AEHZ2P_g;za@!fMK2bWZ zjOL0;-IQ~7kM5x*+O@HkD0YI0CF<_^2hF&Cbm}+>H0Ez?JVmteX7S`5mpK@C)$zBl z^0rZ+^mlF(v-UyW+As$R+C$9L#fkf9(CK$czi4|gY{&%pogf6wPCw`^&Rj_Yxv^; z=4ZvVhLHA<{vEw{nlMF;Tkg5h;OLy?#cuz!^{j}qiM$XtY*qMW1KtlIy_W78#(Ej| zJ4SK~X!msLC@Vx}z4c=UM|HHfX{;^5;kJRhF+Vu3b9dICNsH=Uuj!MuFl;#`CCCzeGD`r>`DH|di9n9tX+JrcstM~F@tFe<(Ul1P$ ztA^Q*0;SK(5^jwKFLzVodZTaXcnug_$e|7XnRwmY4@WQW)^~ScOqVm>#>;F+F%+a7 z6Ry$TX@2zP9GR>c&Fg=3R%)(E#O3f-xBJOUQq3t%<`H+N!^5@B1|&X^&<E>Xcc zgbora1vZzQ^1s?c#-6mx#LuL)QJ#I4;-?z2D$Sp;L1$Cy(y(A~GjnR9o(i4m^ zJJRruAxCVseB}8-jjmS5s7bfqiSv=tuGi%L-kO18sX=W?O*_uXbBQqp+*bE=b{@=H z%HO!;RY^LF~;LFwlS4Sq(aHxyuxFC7>4z^d`}dtGtrY zhsD)jmaxbuV7_k5Uj$qEA%`I0AB`aGolCHMMrybI9d@*IWtYk#!!v}njDF&GyW634 zsx_bAb;%!~^3<5J(w6S(ka47bJ$NOKX~qA&q8!ZXJ$u%&s@X7gm67MkkiFxl3=8b- zyNA5;YVwm>xp{-2WG`K2_g%r738ia;3M96NomPgx*55L79R)}^{K_nEu`NDGUg%Mn zMp<=1Jm8t2q>$Qf*9OH@lr&%Um-FGJXXD`4MJhxF_ zu&HCKY_W**lXs(Zy;}+m3^U3^QINss$8- z<*AAwzh!+>rpd4Ax6EDaX33iUJ9)cAN!-+KEfAYNL99Jx?eUaz`Cnp|uVi-)A@%3|~EJG#I>sshvXtIlX6KZ-c2X{EF?x&uLl zT<9gDI0%+zZ2T&sK-A-m5PS&bj+#*hn-;~TH51@9?;stm@5Gyy5QwN`Gdx4Rtp7Sce^@u+;I53S7?bBy%^BT; zU=qfHGMv<4ny^Qt*>ubPs^OBeY)G0%(GwSYkV*7-7(>zi!WuXtxesq{JXUGPN<%0` zosFJ6Z55)rNlqeAgDS2ct?w(-k8qmj^tYN*Q8DF3LP&qVJwG7YVdo~Y7KaHwwq*G{ zd_%78ica@#BT70g^?O9eK@+Q@4G3bhwNr(7NXXWmmJPsvlO#=_HRlRVc(XX?mrek) zPoEl8*^XIl3NpIflE_#3a0lDLt+tGuMUnqT%cF%93Tw2JS zT*>Um_11MeSKvb{#*+V_szp0~av)O3w!1t!9lq(D531Q}=w(s2|6As!+-PB;G><)W zk2&nG{ZhGHB?itJQT0DW(nJDsCV<8Ml{Q31_p%6+wacS>G*^z~q++VA2 z3F2=xD2Gmy7?jdFG@UAe`T;7a*Krh?m(BZ)AoBgrD$8mQx+CX^r0LMHoM6a$_9A+o zRth`F9H7E;Undd)vd+3{CKfa>fVrv;%#6L2u#d7FPj@00n|{6_?6oWWG5xNKVcV|3 z_+zeE?AG^wa!QeFS1todQwp-uxs-t^t?Rs$!21H|_mr>V`RT7kM2SIs>)6mEI9&4TZRxkfq)nNTwVlzkyWm-vM+DBcNl%^)FLpaHw ztz*LO_HzqKH_&*Idc)(5UM!YP|H{M#T?<3S$2FB#t^>zSWVMQjerLlLeE_>eF&{F$ znBSOPa-22Sq|CZn^1O=xiCVI+AGhLJJ21<% znmGPCy#oD6Vs@enh1>d3cjZ{#Sp^gdG@i*5j|&M|klM{1=df$z+!w|8`l1+|>42 zoS^yvzB=&1bOT+AMaBWkkF{P8VP_z5sqQ9rfzZQ#TrQfKPR?EnZR(nypomIff3U@i z-KpL3lvM=t)Fo{7-#zgRm_zo#);$E39clngWAA9d2XuQQ0lGSmVTVTw?>S@^j)DV(vPxMZuEy``qm|L;jEDwyV$w{RaRM}dL!?FZpA$32iMlI zFj#Yn?}bqPdJAJm|2rSUhyx99edSV!2&}B>eoIh&@wMIFqbrAOtPWwLG7V%Fwe{$1 z2ttGYHXqHi0#EXk@rEE$U`d*kjD)GJ z*o=|fYZn?+${N-<8TV)ai7cm>7#w8K>TFVT?i#E!87B2D;05);W&ZVVYE) z_3F(rBQp$lI#;`XiMTryj*#>vRwYMQhvhoyo9;`&w?ul9=0!k`myaV>j>Wl{&w^DJ zpmzC5;)j{tb)es=$Cx^<`#-!oz9)}fowlb4ug=EZRus&0EZW&}!YZmWEIg@FwPGxdq zct3k^KXm&CZSPZthI}UVs&doy9wN<~JwsO#+AfryaDC^^CYT@yD`V@vLwz-QVeG^B z?uP{4qm`T};eg1#ZS7nWbQPF-YPs2$MT+8$bZ z_Kx#at;b47XM~|O)RtJv zLTT$v{VH~p;8-VE+W9eK^Na+4?1;uwkjinn!Y=`X+wJ--4I#B0&e2@lEiiAWKtWlh zm%NI)usGG}d_M;Kt#(OyIdSsRl0Pj+N?(x-pGp2nuNTTQKAReYb`R4W^MpXKEI;%a zbl^!ylv`>a$+xA~Prqs7PY9naU+|FA9wXdA8@oPXzpsxQ(0vNtgh5-V`xkCpt(Al7 ztDZI=|IWu7^W)gbuJ<3(elT-*=z?iTd>H$Q@y;R6q103o^3F$n-Nh8=8HD}Du8s^N z*Gaurh>@3yis?+ymA3II2makuloHG4}r0>>(?L|9;?1}w1ZkiH&R~$Ww zfUO`}-=6}K6eNGBJ|yOEaSty(#bJRAA1StL@^4`^dbN8L%jh0RBXd6JL5rt_GA8I) zhIP-XN)etSWL*pDt-t|WKFZQ8q{b@h%+WBh^1G5daTK^c*Jn(`bXk{jMKuwk(&uVp z+F|Hf6b3|iil*~^L8TGYoJ)$Z4h@YqAH2V-T*5~*!lf=cFlxjFl{c7w=TLx||6!)0 ze|nt%9%ny(K^#^67x^169eS#`(*Hy%z{&Wq97l{xvcA*4H&C76&!3oFT@4WTiAm|7 zV7b(pXa18Vh9G6%8sqVJ&9=VxwvJmk-}%yYKhkm)Gy@yZwTFY#u%bbTE^Aj* zg>9ZQupz<;#Ig-m@Ra~FON>nXU|L{lxtH)PBtV#pF9&dbm--i)JVf+{E3HCUU)YVm zei4rF^f}?60J;L`-`Ql7R0~{-B1#1^NOxpS`fZrSZKuQigWJ*q7YoS3mE$9c-rSB% zo`GXOaZE|Yfb}1(tD_eG*1DSZZ>_6Nd1sp)IXh!53Z^OJ&mvEort+Ip`?-bw0~YP| zB;14z8J6#!H#mpcd+-L6P6GVoqKsFoZyzhj2H{dQOQ`Ab z2F|&e|B-ySZRamD@Y3+QOM@S3n+3>WP`;D)J)b4mFds#>22U=oAbHB%nwZ89wYRhz z6t`}U`KOD{7wp3Q`7MVcHEXQBih2PvgQ0SD$UJ0l5h_tmU%pKyP;4p4AXrfz2; zTn3!p=iCsJcE17^J4X%wAIT&9O6x5sxvw5AspYr{Lo`pZ*a8Ywk3u!0z;E3DmGKy) zFtl5_1-M7?%Ds$z-on%~_r4t;9@uhB`~XqqE2aKB%8Q9MgXIGy3*wpB*(*<_`TXKt zGRkDqk^fL~ZfG5XK0WivCJHJ$zVE1Lt#(>QQi<+Ii)19vZWFP9t%D86k$&H7oUQ7E zID(dUm%pF8H7;kWspeG-hF z2pZH)FE;K1nZKgEqpt1Za(CZdItEg(o>KXza#PjiTMpu5`*plE zOmRP@bQneq%uF9Ws6_=kGh{*j6FRCGIcbT&waVG=QJ~IbkBURmI~^iFq5vU}V*57! z|HFnF2XBhBYIO~Q!6uz@g--2L@?R0zx%ve}8yJ^aDyyY3oX#!}6GdP=%ZKCQM&T4H zaqLS6=ueore6KVL=Mt*OH~5mEPu`uVb1C{z^mzY-w!Cm!WX_HWouz?Ds(~-0u}4ZL zzh`sgLH;uE$awgeAhE=FhEHW_2j|1-zY@zcJfeE(#_5{8GlTZh9&T4@;<3<+1&a)O z#8l&Ao*e2XDPbxm(;5D)-rEZo^I93Vl9Y!8gbsPLKF%g)w6 zd79?jV)u=J*D8Y4eGv z8JhyonQQNq&A#IfVZ{jeCc^kNP%SVee)v^9hreOs->z=5r2Trt<}{)Hq%r2${mfG@ zcI;N!l2flEKcN*Rs^GGjR)7aVB3jc92olkz-v1p@)NXj1<|J%7Hn6~ygO+b>6~>JO zs7Aq2Jy2Y&@k(I9dKjmav{TCBW+?edHg*dNK5>@8%v z?Ya!Q6?Q-jwr93`c!IEsHhx+M6*j2tOe3aI%ycqz_k@)qBL1xU9RI%U`cC*9HW$u# zFL)q*#=!53KgMHiDw2}|TUh=|avieQ*G~{LZOeAvsb4*M;KyB5E5W}9`|VvF&3WR- z4L)ez_Hqqf9(vzuv;~@L;O41e0elp(BcT5jBNBgk%fL4#u=2O9^3nf6-&IcW~CLv{f}Y?fe~)^@SF5qt=mx%A(^{Et{pd6Ci7ri=M>EZZg4NL z*gt{{FS%98G4My*Ie~i-sr19{iIM(yr!m+3EHwmVw=I=?E5Fu%xBHPE*R?9K5|eC#q@2)_zxsnTPqHK_vhO&E zg(8h`n;Lf_+@|~Ga@!=M{V?!#;Ihr}L~J61M0q5y++~DSc-t~`L419&e+~KCp71+9SaH$7J1g5S z!~2mE>>LVOV7pTEV}~Y8evDG8HQ}iQTW6)|-X^M5j%u%9*Fm}~>%=M1$Nvo*ETJR%pS+k4f(SZjk}?>u7H)5J!@YEF=S@w$5Lz5l5o53c?FhGZzK{{K|u6+YT}AUD93EEQRmBh!peq&QB;A)7PR}Y9nhKiI|$dQohou>CS!QxJ6rh0%?a* z+;LJZ)+#qxr7O1|Ap_{eB&Nl15RF3k8c7V-!&%Db*P8hUWa&$e*dFMNVe_r(f^`>= zN!6vv4yM3{xWe;4j*fDb0u?NwYa(eAoaWyj0|g#$?!NsE6r6L2-*`$&==Gfz*P5MU%pxmohSuZO}%(BRyZ9!I4bv?jYqTkrdX6}328XprMz}Sk~Wn|6Q zt?cqQF&4B8wp7LHaBAWrZ`9*IO6=OjJ~c(V3gS7;?s^2P$6 z0g%g0T&<7-*&&S*={s{AsXa6Ql$?l7BkkuXh==weD_k0P2QY$N61&3DZ|q=ckS9bHs{Xl0|p1q+yNQEYmAjcj!#Cf$EJJ z7NFNLDfjn?A(?GGXKX;tK)1)|WBeAzZUI|Fye+lwr_D$Cft2ar!3?F<=wivlHNzODO3<&!27u{=K-2@kTTHWvQ z?2~DHM360K);Cw1m0L~E(=S?yk0F8WlVoy>jR$QrEgGx;BDr?-A0*eHzYxha88FH3 zn~B>9YcXvAB(f^lZNqW3@byr!j1d43-=+1LYW51n z8r11?rZX1w;()`@kpbVjJYYN#KF}FX{$PZjcvWd1WzYkR~Cu^_@Tp;yKsByLsr&=u&F~1BF8m0(gL~k-nqoN9-BR)ep11*PoBgE zX@R}50|XfWqy9N#=B22O2GazsmesxQ#)}zH##buSz14-c{-GBy67o5Ce&XV$^}|2o zRbxGt*XwwJ;X{PPRqTk6xS09KUOx*+;u`+O?<7Me^jKWZYeJ7(%nc z&f@kyz}_IGs3wXA=bpS!03fE|8r%4+@A04gb41d=*}j5kuI_7n_}1=Pp_c6HPhxTlebRE)>Q zF!crXzaS<~+CL&#bb4~wjIbgJS3+P1e(-~t)Krqmdn!uHrON^3atc7a@(4~ML0H~?vOM@!xb8z6Mm|zTxR;XsZ*g+Z z3qnNYlsxhAA96-4T6p@_4%j_8V9p+q1YXnl?-}adL$i;mOcl|&$hWQp_ zD^31H3eG(_??2oI(;pd1!usOl7UJ{5>_mICFa~e{D(MI$idxT^&UstFj<%&>JZJWp znrh^<8R#GkF7~HoFlSl5%}UAVxIc)N8u8@(c8Mh2Z6jtVyzVwtWV97VzWklv`i^h{ z{KJ~XJP-f~f0Q}S$)bp^H~wH_e(OaGl3Sbj&P_Y=Ju%VgTf?I_c%+-VfI;}(4_iPG z)ooJF)`$AQABAV~=dDiXmyu|xmSHs*8r?Q7(j4qo*=#Rt;~a)w!AeEiHZqt7i2;;3 z{{ndeg^!SzYY@oXD`iXujt;x1FSeR)nu+Q9m&bkG!)IYMI zbU3?c)>ia*fN*a|aqQ`DSMkZi&#die0Q&BK!Le_fZJYKVzFwqNdg4bKcqMs2#$fGK zXGN9Q?SgH9ulI?vz)h#+zRQSRw#98d)0id5?1TT<5y#G@ZnBbkvQSLf+{R~VYyLRk z*|EZ6DvA#aCjdVGcG4?9`=aDs3##hp&?9Sw_S;jV?5kC*EF!+xBYEOKACLCTDLN2BTuoh8 ziTjvU=Lxm(-uhZTClx%+e5Pp_TW>cL(x_}fOqgWO`e_l2JW6k;j0G=C4G9p{OK=k= z*+5L(o{{$T%#{rV4$jn*Nm&!;=>aWu#S%zgOACCaCDqQY(05Qsm zF6)Vz?A_gK6Zd_tN52tJKAi18JHj%-^e5uEH@-H)3l@PynazX**qZWE#gMkXA!%8d z00C{!O^^HCz7XCd6f)}s(Q1x}vcXJW`*siZhnVwUYFFiUF|ZHj!3X>_^@Ubk<-SzD zoWKUqe~_5&jiho2?|?JT^AOwKL)Pa|(&%C)9Re551dWgpHatm*)pC}Y%*qI6?CDKH zAl*P;6U^rtiDnbVm)}Dmw4%-7_!Pj4sTbRHA%d@2Ww0TVxt!M7GbtA%Ehsf!oTj5n zv;8pwvRv;RwxgqpfTsBVR*KI*|HgUFcB?ba{Z>8=21}VClH`C7&cv*0iG)mqpiZWd zme+mn=_sriEBq4Qa@hZ~7r^dgfrntY<_#R5B*r_Rm(k({fPTb8@-e|*^0Mi%PyV0N z`2KHw_V{;n~mCqt2z*iv8^xNal1e2*Z@uRaYVsZwzc z^YgkC-DpZt|E_GZDL!mQC+xxQ^%s9Ka-(o(OXVvLK(ay>6r1cfd+q9i<$c0>H1W-D z|Jy9ZB!TgK7@iK`Qu+&Ix?}PSgVm(W*z5Q=(4>mM-IJ1A?KKS@MW~e&+Gz1;d_2Pb zQ<0`h(Zl6L^)E?GqT@%s_4JC!u-8y1?`0Ivon;GngO;~ici#9}_vqGGotrfk&xaM# z=&R){?b98hV7CJ^1_ub?QuFCfZZM~;t17{0S~6_~RCC)k$-e0^@iceT^h(ZLa^J9b z2lv9|1RT%l%e!_<4uG4MC8TrRe^@-LI+#dAz)=})WIPA)vd7hfBj;?d!J_oO4a^U8 z9T~lZ)UF8(5cL-(TSUY}pMc@PLEKSlv(Dcy`23 zu>#S9UPq-gRJCX)gKB{5h~G+3&O$5oG{^?PaujNNP9K%fk>Q5skLf{(7{dYI^0woE@MQ8^%IRgs9s;>2yY7Atm`s(brHCx4c=!<6oFA(z1$b{U>B zMh_hEmLeNTN=EhkDIX?v{mgaW2zh%^MXZyOse)7WyK@Fbb%M2P1Sgx|gg{dqN|2tl zL+;={h9QRN=9qcMZH!O5u~iGH6HkA2b-PFsgCzJ3XdzN}HET|$`#XB&TkkTu8@n8H zf>K}2Yf^g>Jy4RMkzRs8ya45mWI7l*7i*M&D`MQ1>fW)tPOH6CAdq^8D-X&yis_Jc zWPY0WvsSK!LjAx9w{USh-=A5^lMM#B(RK+Gmgwr=Pk=J9HTBSAmMFVQR`~A-1Z9Ii z;P4EqJ&A0>v&|}5Dy7tv5{a}7YWwvSeA2k5reO5vX~8{Z@6X{=a9jp8fusB>JMEmm z;;ZFU)9?KDyHl-HUl?1Iy;ll6O35%yBlLn-N!0t?u`eq%U=HZHP{;OzWG7L6erzjqZ|3_pusS)Moh)_48|* z-n>lkCkWY~n^|V5Ny<>H;66v;?mteHHic&9P#C^WaaVNHg)Fq?NwX<%R1!OnB0MHc z(%@K%$e-HG&S!9itn1cZ95lcMb@wHlbBXJF5mgAKEy7G&llPPtC;b4?ywdMYOmoYw zA}&2A6er}Qn);gidFlrebpvSS%o6$JR)0^$zh)R=ieuU`MTn7DidQPIA5Z#7*d_(N zjqm}QeC3Z3NdN)z-wNWz>eAg7`iBU)AMGYf1r00vHl(Od zaaGe=R__6^asNpN8+&{C#r}r=*{U?LQb6ye$bW#!JMXy1Y`P)JNS{BQ%=Py|@-`b& z#iv$`7@q(%29;04R+2(Zg*^6Q~BA|GC_fX|4 z5~{A??{K$7W^9iR6u{5iW>=f2hpXV(XVNaf>*2ehmIogL<_}J!Ze!2V(Yl<~)&8Mj zqF2yTcxGaZSX`Vg!+V;ZN=F?{RGEc~ln3rQM)icz@eDyA-qpBEYuqKr!P?ZTrATv0Y4{o-M*RU)LDuEAyQh z<%8nCIzL8D`TkC=43kx^>Ol|NcWk~zlH<;i4;kz%G*1-WzV~M?eQfraiX3ZF=JX71 zPUCxBcW+T_7bAOy+p>qqlg9EaE<6^9*B)8e$F59eG847$f8qT=RFEDzD|C4S?z&C7HyR(#ILO!*=9iBnf{)0HIL;eG*XFWal3o{Y60DHPO_oPxcpRjcdW+7rpf2)Kw_pr*UXC7(9TxTYj z!y8P(^y|ygL>e-dy^exv0PwnH`i+SvM%E?VcARdC)^X_SGrlvfqaLMcLSrHFxh0N)%^Q3dxMIJ% zI*vSX4c+u>o@c-4r~+tlo=te+)316fsNM(TF-gC&7i)F8wnu@z1hHRY5(1#2F}u#v zy{rPabu3`OGXYRz4`2?R`UcJE&l=vknNNYF)sVL8_VhPFVA=!9|M3r_@BgW(j>faA z4>4s*=++&gbdN_NW9x1*kRoyLUwl?Ye~s3Ea-$yGiZ5W|g@sc$-}F0vop+V)2wJmV zM2>4j$UT6Lh6^nC>sqj`yh=Lfu_IiHZyfh-eIQ2j=%v{C?v+8RYMQX>7GYkmXlm{suHjpJJt;+gR57& zkC}aUbk)vqT@GC4v4YCPGf{8aP&=#xY232mtWq<2<(i1z4j8@p4XL*6rrJCjWY^9JS1Kik>NO9`3MvL0dTR8V?IKKr?8)nO76(&0qOCgD ztVxE^B&(5)Hw@#t{|AjFL2GdzP7_!XoH_3H=2qiw7Ly2;Hf>O$p$V4(h65bla|8Av z?~)oj_zDc&Nxsq=@EG_FqS1Fm?l5m&b&WNc_bz@eU;Nf_W66)Rbg+B#&BLMLLr83G z-BLeGsqL{%|5%#YQOa5HOoCChMewv!A?)QKbsH?#uKi{;qhJ2}#xFySY@MYgnx5k2 zhA8P{{(UC2p(AEOd&JLKbo>sdeQ{#E*%gmC8*D7~VYoBSOx4VAo7S9j%gNREoQJn1 z&b_QU&@@wkdOI4@i<>s#QSY04aZL@J4>`a(Sbc!H?flsTQ=!5gVJFBPA*}mzUUKoL z;VA0KQMlhd`eVkP(YO=Waje(HVRu65CE7ex7z`GcvQAi2=E(m`r-9}a=S94$OfiJy z!$yO`Bkv4^xcQ^%Ok`qDg8Cc>jJ|NzD$ltD*5&?Z1ZKk-lKfo>s?0wARfUMM83L!qmI}@7`LoP zF?Ay(!+4#PJNGu!Pq@lB{qHUg1b)4Gn!Ns;-}5n8aAR@w%%^)#Ll-7rW5t>MT?{ZZu*Q z57+uTmAFmq_E_`T-oeVJp`HF`+5!Z4-J7~1k2D3&Q?sb6N<6a6T%4CAVeVu}w7rza z+uD`#>JE2_WQrkKyWwmb20!O=3OV(r63gmci7Cc6$YKjr{T}tOvdxV{g>l?%d7Uw1 z$)O}bVmW;{n|rY=^W9%Na}SKNM$RjX*7B#M0lP3OI=2fj=<7uHJ!&|2;z zrQJ$Bb_|%qrLiEIDrvBf5qhL48A1B<>-Iw9SR$)4od0N~hfcG~Uo7Jr7ye-V+J1X@ z9*TWV;h?#&=Fh_eyUVWzJYKDMugQg4MI+1x%!wEnDdnbD$$ya<&1(4?oW1k<7M!K+ za!FkmrOUwaHL#rd#3VuIhI3v#^&-L`ajF25RU(H{=naO-6{!lnZT~eOHh9{rzvh1y zfXF;wc%Y96Y*;q7Q_6YqY|kt5s;Td*BZ>2GVzl?cgh=;Cr2MO0OrcTaA~_W6uDDP} z&uLLVB36Qj$Odg22EeitXPVoBvzB>ChL2p7}MF z3KYJwTx6&z%FDd>o!9Su=Z`N+skF;_a$#;0od&(oF%4@+Hd(Z)Ro zZIs;~tzmt)JX$wG^L*I)?%u#6u`u7IJ5HlOgY0gpv@X`ZMhm)Tl!N!d*po`;9M`ig ze~gI|41J_GLw|G(&&_|VmK-wo=>v{ZJ2B;-ojD};$Lr>`0X1L9Ezd>!;qSJb4h)Ti zUY#)puaBf=7p$8OErib$3`VmWgj>%FURvp}e0k!LbHq&wIn zxXA9YY2%gmG9nxCATig;A_?*X;WRiQ9&5%%f5+@rJh##4-LEs}@_h8>n2Wu|5dz{f ze!f^jsmj-GpV`5$YJAw|^(!VlpHG9H($GMb7Lh{bgH-q7`_%R!l z4&$@?yi%-zZTJ=*v{uuXhy2r3Za8!n;#wS;y>RwLE@?yJSn9e&ciM6t%M)NPr=Z_|&K9@3Kj>eM&#nNS=tz#(nJz>3aif7cj8uLybO!e64YLWQ&U- zCegZvv-;xKzN4PpDcQx3g@!r(TfRHBM_!e7#(OaXZk2j;bimT|=eBli#U~4lOk})eG;M9C!HI$GO zT>m(e{Vy$5!t9(89T*k(^#dGeFa;7~$kyukwcSTVJe9*ww0|@K8M9DvQHLRxaM7fy zb4jmeoDQ6`HB7;InPii?2*BS@GJMCS=4-9S523s2so91KQN-bHwdJs$D882X*!Q!6 zH)9H>YlU{T(3-=*(L8a@o7Jt1BKQSKH2gIacOrp-0nH9`Okm^Y#|4L1mBUp<-?3SR z?UJ}z%Xyyly20Al1aVe$XVa}OA_%NLVB&3~XjCn~T0!9Clt~erg9IPEfjG7~pBt1( z3~Y<=i33x_rB+X%%uroLnRe?#=tyXBbC4B6D^$bKY7wgTe7j@DPJC+~C0XpQr)O7l zfbXiqIU=3XxB=)XlYTD2B!wh6qNfm}`T!+e)IGa}#CBVh8b?ww{9O)-=%Fu%UQ7;R z&Vh#NgUd@*ZwjX~;~)hklSjsg1xHIOhTQZQn; z>LqwsK0=7_tiAWUo~6EM+d31Te1B!PSxi5VsqJZGZ;ZLu zUMy|k%G-XoNO4v27;&-fUi98x6gar*dKBnD;QaMKln`OtKwzfHETCw58~%*f6jk=g z(SlZnNl+UfTO8$vPC`y;2aj*Xyuw9m)h$jvUW~FR^1Q!1Vk7+{(|l9GSi}wj~>$;YrkL2bobRAG{&qX4u{>D zNIbb5j>E`dU}0D`E~;yV?^y<@j8YisCN_5I;RdM-ULm8a;!JR?a;FJM5XlBVWUm6m zm5sX?)?gU9pXe0lsvpHyd7M}b=;+B3OMeI9=8l9bYpc4w_o=Yf__~w zkXGaUDuLf;CVY2BC1yn9a`%d9itm;7Ld)BbfZy|tYyK@?t>eO5ga*9HKh-pPAA3>u zKLioJQc$qhmZ>5zk@Z)*;hjnfcJ?*DK&txUN{SzYqj|=+iZ3-FPP;RgeH*jANnUp! z1=^3l&`jvMR5NTWbA`50$rqscX(~IRP&HpyOC6d6zloNXqvhGGbN9-H59G$C990c>A(+$~B{}rhL*nC1SwNXgCCJcpX;F@xA&U74 z%8Sqgr=W(^Z1ZR>;qZeElm(w7&u9tyen(g6-nn`5{M*5F1L)Tyt{^h=cv(ArggrreMi0VTB;qC+B>9PII1dm<~hjh*_gy&6XInK^ii8H`#K$*V`3lD&Vc-mUF(GW>IR9t%>NWR)EWgg)Gzsxbks&1-s>K*o^r=rVAK&xQ33`0Jkngz+P_uQ19FAx)IhZx)Kv zIcI8QSgnrleKaygv>{4E!|33u03);o@1oRfR1&J!-9n=w#fE2#JR=Uu_!VG zBrm)6hRRfrZ5Md=e|(ahHGWl(2eNUIYN1f#&4-PCWuZ1jOrPJE*4EnK_ZtDmJSWOu z3t(Bf_e~cdGy4Dk-ZQg`0@nj8X?Kiyg_z4BZs51)$Mh76eUX@lV-Rg(%J*Nq)7scs_ z(gk;d~tX$^F z?!veJTNl54d=OE`tJ*%dh!XpwCVTQT!+g))82o24Ap_kU;*fC=~{oNi)2~b&n}UQ%3x8F?C$eyVuj78PjXQLMc_3SsKNy z*I{F4kyW_7cQ1X)^n;Tnn4*%#lEPBkVw()9LzrF}YX=V0L~yYE?2M`n9(9JT9ztbn zYWy|p;J0ebU8fEeHj)Vk`T_Rh3ZA{=4sKE-rd#;qF!cH7O1a3he$mEW@OqZR!}jpO z@vt2#aeNo%tRy+Qm=U}%{E6?!^|4bOq3v#YCgxC&pMZI3diUMSymF=f_ggxiShOC- zd7L)(G1>0QpA{a(1D+ngBVJXdn}4pMXb9a4mF$(HE5p!cV8md8?u-~dJz%~$s5^hp zwB+Onbp%=Bb`N|+2GKlNlfmTF}J zs+6J@Z%M_lqwToe#`4A_1ck2KFVVA%%1%rr&Wl)PGdq4~;=x%LSyv*CB&Y0+3_8e0 zGBr6l2?j|jD>4=qsPzk3BUe_NWep4fY?g+&$cG!xgtnFBY3A;M(XxYr%`#?(3K5Gc3)PRc#c!NPXn%1WE{2_(cn>X-F76V7M)-QYvIu8Fp#~^nXI|^yVeEH$nNP^9qi)r* zHv~3|>;>4C_|IuS7VHY^rm{nS)wO+^SChxw;VEtXVxWC9YdGi9s=DhkWE#Ou(z+#z zLIzr1Wo1QKZ&j3j+pL)+U~=EL!u`G6B^%J}KJW+kSqDLDv&28O+>NLv_X^(aPv}9= z`cDVz)?ZEB%~>7URL%23{FbWVJX0dz+ByfdiSR(fXxHWO9Sq|OrQbrlY`2h#tIAqZ z3uD6VYG>>j8J-5jvO9S7_YjRsMy16jhsI-&Ry>>((pdX+J~;F(n4Z>h+Mh9UazMeUDBJt2k%n}}Nt{x|Bp!_50h;XH=(PY@u`PzaA=amht5 zH*RY&`$vI`gYb*kH2i0iVV&A`V2z4<{3uB3*a)E4+nh=SZrI1-^cpo8f}*XC!DyWJ zapjDbw*(1ipW`cIYU^x|Ii#kA`=m@|k8Wo9t&rS)-g3ePmUcS`0$MxK`3H2!t{>XF zNl!;XAAl0tU$RBvkBu|FPaY5(((7BSjOj`YuA&#Aq0}JY=fp20wI)27_k@8wOXR+#?5H$g0|lU0Lo&}_kZ(c>}9gBbW6NiR_MBOuprbx6WT!tx`CbE_kQK zwuf;WVc%?}cF8?6DomI85kaTqAZ^iREfV*vwrc|;OYL49>J@_-`aaafgw^{qx zn{aQHLG5qrC44mHlhoO&EwK*21AbYd_C(33Tn)r^c^QA?KonmE4<4q^w8SXu9X$nQ zN^`QYv4AI;wUzt*z39~^@MVG`GnSlZa5`^Gr+Jv?JU=p-N>_-1_s-Xx%vEuecCj{G(A!p-_1+G|g zS;?o@$c9JNleY&W$JWCr@taoDkD3dv=GIlm0ur*s&$61PKS19hM)s?Nxf6LW1ov3j z!#5kBeVnYL4_CAg-4dRd0Bz9@9Zd9n8l0huA5l9{MYn{UU|<<%Edg8hF4-m-{dvd~ z)+FvY6%%J!46-TgPVr~U6o1_a;W%53f;dN8d&J^U;<0( zK&3`>k`RE9p5H_fKL9dx`2KIyKpX#)s&a0V`GYY3h9wW~KNcpF0|1a*XU*P#y`sX? z5}+SrBpy9vE@%d&?{UU#z5Vm$Vf}9tk~QD4)~4u@9c>8}6&q&f4iXElo*TW7h)dyz zWUX;++Gox`09vT7nH3bpqB6PK1yxuG=$Nxv&FL*iZK;w!6fK+v%(#G+reW&Po5~XE zn}m5t`_i};lEXY%-4DhW?4ZWpa;(}x1a_VUyISxE)Xp5MCk}FzD(8GXA=O$lULF{y z(P(U&0d+m^F5J=Fn$ZAKVn@K~g#;S%e-DW}BUayBb#4^f>4`bF{V4pYQQ=lAW2%rR zeu;|LzIU5Wdz{gotBPY`J8(ysnr%a}xci^HWXXPP#}CMq7*1}f0S{I70m+)ii|*^F{1ECm1c_H!!&F_*Mgx6iL?E9__rGX$lr*o-vJ~~7t$SJ*<-{DG z7Pziyyua4rT@raFLs@64YFZ*JZI(n~8i6L9SqNhM5k41Ccm?Ie?60DjV;`p(Wi~$Cwm1d8T!(pXw`FsXb?8KAV zG>9uuK*eS$_r+Ng8sJyO=BftTwa~(qxsb+jLtYwcP#~AwJ_4ma&xe`73N>KGklB&p zhF~lvw&-GG!SV9r{0F2f!RCrzu*O-}3(diVzh1(BtQO$Uh2MnrB|JMmZ1W3)$2R|P z(;qivii+J8hMjRJl;YF_N6jZH2+9p+NKf*en+5sHop7s5GreIdsZKp+QuTV(FB4XI zbKAY^eIT~PVrZxCdkc2l>t>C>0{)3g-`&BBAcMI_uX)E3@6v~o)k6-NVYd#coRCjF zR#WuV&xECuuK3w%F`gS%fIW&n$(o?s90(U$%bairO2pL46 zI=yq%wD<;|?xt7(!zpWW+42C>RbebPqeB#D1BzfZJd?p!G54rZ{eh4reF@i?Zg7tz z=Lvy3?J%2G5D_lGdj3C*#3u`2BoA6zvp4n(hadVj@}G8@k9S)U{1P^-6fu*ZE{hr| z(8ccIGQBH8ztAXoF?wae=D{}u7v``E1cCVRlYNijg_N;NrDpDI#q8RfA?7$Vf-^waKIh7Php<^SvT@o`jeC&A^4-R+=K%}!EW__9y4XhQp5KO3c6iU zK!n*tF@7-W$nGTP!O%IrS^lJ$L{lKIMKV=ea~Cgd$a%+GqS{<&_il)YWeT5MW5E|R zqU{w>fI^kL0$VtgkA3nKNqx|=rZ;dbx1oY9I7f!Zb4!KMGDcrUf9*4S_GHTkibxXmq%B<#43tUNMMC;&}C;;r!S%>y#lf$c9Es;a^OJJ>4NJ zW(+0Qa^N&lz)#7Uwi%oeN|19kBH7_u;$Sc*FG^w+PtoJ$2`T=iH-)Ov?F20Jay92! z;F~bm>UgD)K&(ddX(L;_LoJFTTi7Wme|_SwLr%_q2;dhDgbYZ&y)$9Ad!o1>o<`fP z6zfRJJkGWh8;%{Y_@MfM!)aO}r$-7lUFk~vUcErP3>sf8R@s=qUi(>%MD|r=e%I+g ziAg0)qNfs-vj4ipT0AgDAi|7&z>K(%Dz0o2foDs8UNPFZ(GUy*WNIn2m z04^HdE1mU~dU~2t_SnUMef9O4dtB)Xr~u7s7AGQpACkN`;v4rBV`rnL{F#q49Wd1D z*;!L496|c6=d_jJcW_sClL5| z>QzbK3K)nI!;3?!vo5IFcgJ?ENKgOPGdHw2VFQV|#`Y(P26c%e23mof#X+f~7c^mL z->usV3zN)?;b%>^aC8q7^(cCvz9AsX0ZJZUbWoR1zUyI&qMJ}93;*UQ4ic;%6`{)#=*p z7W`64Oe3h}TzOReJ?F(_0C&7i^Q9ZW8-3cy!!AEMS(wnZT&yG#8}cxb$K9Lo8BCC73p=a;cnfW@3ZHY!b9ecTOTVkvi`1p|Xps0r~ z$ftjNgd+A%mM-hm%J{b66#x!fpuZH)KQIIz_9qR#|2gRo-uxS~{4IaBDhebI>LM}b zl@+(ufkw|vIwG7}Y`vf#T{T7*4wph-A8Vnh+|_zTdukK*OibK?Ovd%r)CB zdp1q_n&d=QRI1;>)w?9Yp}Xx3?7n=1Xndo)j*GsaL)1&6?CvLHqoEjDJ#{0ObZ-?# z_=et<=!6rQ0svEG*ob2j@_J1H+Ria_z+CC-o$WB#;=NBZo|u~McQGH|0;?qE0r^O; z1Zx;Ki%&KD);^=~Q?ZGyfqWR%#fm6j=bmKDQ)qE_Ac z7i%#m<<&kTRyya~eN1PlTnbGc#Hg)@_X9MC>YmQfU(>wo%kF_@}4 ziY3v``IiwBEV~3%K`Lm~^6-pEP>1bG;v@P8z3^UjhG5b0)sH zO%{hRfK5Q3yc@IqCG>;2^+G)$`)k^a_qS#ROe_ea?Ee$*`wi;^egORJRU-NGsFg0QtO_lL0SnAFC&(7h#pP zvBw*p3eKp#&tC=@%LT-F4PY$xN7j3sZ``?}dCrXThGn0yuKT6V>RLCRtcsjxm5|5L zkKX>vJCR0)tr@KTEsqW%(ty!2*c~vd6;gkXI344=&X~_(_6jK$D(~+tj{m|~yG+8W0 z13H-ay3bcCS7Uz;Gsk>4G1Q%kWk@KNiw!IeD~6YOZRpZgYksxMeGX9dfvtw#s`MB1 zcrK7WlweDuZo(N8pBK++1Sn7f(S*H+jNGWsI zNYIABj`}ytf!Q0ES_v;%?`4k2KV^;{{B;P2kWj8Rc~)C1RrpPVcGQt>z~pA_S#a{W zKpR2m)d4EV#`Rx}Wq_pR-tr|{cIE|QIH2$4pue;XtIVH8#Kp%f_n%wV{I|mSv;6QM z*~@>jSpH931S?n{cCfdvhNqvPrl}y69i;-sb?PI<31WV;rp2n#OiLI49P6d+bd=E# z+z$Kp{5bKi;jPxP*gZHmDelKa@4Q=>=9Imtn-Yg(-F3?&YF%4cj~6iK??FI24y=r#lDqML9}@>=1rrFx>e@7s9WIv!MAFdch_xg00tF*<1z2I zr)^1|jYV~TP|i05Mc3ZWO%|a4b*F4#jb9ib&}0o(t@chqQTTdp_IY8^;;m$#zbaPs zL)|7ryDcsbNDk3(k|#92MvuhpID_Bw$99aI*FAqK(wOH=A-M|Y^_7;j-cI(PMdh2| zI6o95nP*5w*T?|wuF{y+#7yR`h&^3%P4*8l8|{;CD@J0NKu7R$SokzG8k-i+mN4|Ls&^g4l*3I8|FlrSW$K z9+F(f;O)MJG}Gq7vVC0pjWQ)jD^Q7he~oFUqb4M0kY2o+)cAx(`A75%hI6i-2#xnn z&!hG{N@u1E;b>j7-_FQh1JeqL;?tbwk5)$I1TWZ<*zzP@Y8PjF1-d|cPZf@PQb#*e zPnZOzqfx?MWFw8+GTAy*(sb8Iv8Lc$aCgZ{Ab(T+-J)7O&YC@Y@>1#$!-qa5K zOq+z$uwMUc-EW9;s@3MjLY!Hrg1dQu=#a1lVlRlhmyTY%IUhx^I_zh$uK)l7nllL0 z;CcZ>)td-XCcmq<^dLhkGuSyVS08n?1*>w%H#Ui|<$2fd5Gw{X>!aq0|2udCce_Jl@PM zA;2|TPs<8Wbjw^wXy@&BKPm;682&;jX8?XqUTO+a@Y3HD3_8SaewGr$2&)FclV=L< zui;nKQS9GVPcrlZH!&FkP-w&DlwBtcr=%&JZEY|-;#baB51A?Cx$WuuB)(Z1oqY^}WlefxHc zqtp+$fYPDb?qfc_n;cb4XR(YtDVqm)QSn$>qekr9EA263i1+exYwbMAegYhTdK|K^ zI$iteSQJV+`_1}V2;+<9T*@2kt(${T|Je{Ke2mj}WxMn3fpGLP_@1wt)ngv?j39iL zR@l%u@J6_MfVjdhGkqu`qN0`vg$l#eil zoePq9OzS^pfCXnaF@yqPk0GMiDoAXZ~Hmg Date: Sat, 1 Jul 2023 23:28:35 +0800 Subject: [PATCH 032/337] Delete visualization(PO).png --- picture/visualization(PO).png | Bin 15571 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 picture/visualization(PO).png diff --git a/picture/visualization(PO).png b/picture/visualization(PO).png deleted file mode 100644 index b0bc4bc998577413c865e6a5577ae310bf6892dc..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 15571 zcmbumbyOVR)-C#rkr09>NT4A=1HmQ1Em&wEXmAaovBsrwNdm!xy9NRTcXtWy(zv@z z2yS?Ni?1(?A*HEqJ`;%}CMNa; z-Otz8?(=_azlYuK{Q1vA|L^|`tD>9Sd@-G$pZ~|FJ-p|HMlx8}J$SPvVNYC4?5!y; z#g6VL()CUdfVj*<)tf3r^ z9y3eX(WdN4Q2yCQ)7R%3VsMllN@P>O<%bjjiP6a@?^CzX0Q-0RAMqJjXoq{w8WRem zXsPveer`&4v^b9XdexC*==fSL9;iW8RO(bw2cwlyzg}TkqFm~E@$@l+zj^aCRN#cJ zY|v9Eh-NdN7x}3&uhtnYskd-A4g2UepGLrYwum&KdgLE%GF<75uMrI%O)W7cogsEg zUyQRxuXRxUE(DjeB5=*;?_eA=9D25eomPB;8TP7@dHKX=-Y&g&Tc}el#-y(DBjUzP z&89OZtRXGp>v7#PJV+hsx(cV)B(|c~V|(T7#|avH4$3)3H_@j9`_VWqs)-p71jD^M zGq4ZRi^xAI6i_jReS5339=`uOL54`+hAzUc$DTunIRMs((7NnWU(V z@a}i1znn1>d7JT7_ovj*5}Pl%0C{&@kIsRw$##HL&SDk<@+~Rwzzl*rK^ks1(fOdF zdMf?ow9^}-OLjYl7|k(fHfDf(ntFzvsqXIMmItgQ86H0|3x?CW=Pvc}O=+=2yM?RM z7j@srT<3TQoBOh@DX}3g-!V@rYa=YbacxGk^=}a+c~p&=aX$Z6zAF88hlT#zQw1Ut zIb=?D)(JIbj=>kNrvbcm%ymZk`gIn`Y3*Ao60NT}ux2B!-FiGIq|STx>*_y4@kR@F zxlXL*h&62ES!v(7|1=%=Ale&a8+f#`WF=2%u3t0KbHwYd($!*Hm~}iJjO*O5lcCX| z`VG76MN(Qc=AP-PV)HC7t zRA*3waEG9@{@pom9$CxAH^HkZoM5&*cFJOep)5?><498kHS)Ds$5cKm z+D^wJ6W$bK`+U%apC;#XZFTa(wHu!%XVk~kG0AEzqM_b6TjQk)!D-G{8njXpktr@^Mw%wEU#mkN!FP!y5oNjYvm zxq#5H)!4dJmRF@s*Sbc5?gf#y3Ac7%W`fXjWRmUKPC2-dH}NgKfcZob6`v z)c8`g&=y0%@P2{z5l@O=e|qd#){P6-@tM|D9P$RDbyQQqp2Wb$akv-go2voq50QE( z2?MDvj&=$c7nhjgHFpF92M%tiAS(qfZYcAAXUolN47meN9v7jcq)hy9yVd^hY^`c* zZC`_%l>a@kw=-M#9ZXCt4degkmWETm?Fs+QJN?b?!6ca zpP&B_iDOZ<5~b467Dg@ojtDpO>H~&f=7X;8?lP>;9foWaxXMoJDLz)G`+Ap)gbkm5 z{-j;)jrZB!Hsj{zo=(!t0xDl}5oBdW$e=apc#$`du{_2(W<*D@(2=7yD{{1&U z3S9oID@4M6|NgD?F(4qIWn=hf{5J`_0Ma?P69^U=f9A*YUQ72MWSZq>vCh2-38V@9 zPKrqUa?>%a*4EY-4pV}#urRCRZFRHBQgqLU*cMB)qHtm`oVTZ2x;OsEJ5xCU^)a00 zafr{ARaIH-<(6|g!_Ha#t*s9{&boC+`eR~??ii!&7q#r-1?02XA4%hl~2-M;5v<<;Wl&?ODP_m22 z%ggsBGtkrT_q^)=u3Df&j}D{o;{e)}@#X2kc!`l5gNG?)H@Hi-cw0bR z&CS)xT>B;}r^tRs14SQZO{1W$ZZf0>9VF$o%UmrFw3Pg~HCdK}pb$8jw(aQXke{(J z9nEd}z0!?x3d^iNTP0auTl---S05wjRxPT&xrzHM>@SvHM<9vMthndpt$MbVw@vKDD;Pkn{QFVLwVO z?AJ?eHyfc)4iI_SE7%^Vp4n#MU1h2|PU%gyX_GS5{nZY8LGD-Q6ui8Xj71&Xk#@b3 z&ytn8t7>YNdb#G5f+bUarAdY>vHSb_Hezy3f6v}cW0#{pPeZP3ZRPm05wPf(MVrFATQZuk070g?DK}pFs%D^4L1f`@LdMFWNPa5Uxo2mRaod2Bx3eQmWd=9%>udG9T zedB0tx1PtZo6B-0hd_zb)6-2LXt<#$B0<+mn!RpjK5&%>G`H9K|JtUZX)zWwhV5a0 z5H{k=I44B<>*t1(hL4CCoGT{e`HLxK2_JiruBLFBxu`f_-{);! zBfjhMx}8wl&tJqM@MewlihDswre_&_XgC4y>aPe!*(w>*>_7bpeA|OsUSaa-kC>{ zIQ1i^f&_yE45TrYuZ{CH0TmO4ejh-=jl*(6-!v`jo+TwE5xq7JLC|qPIXP7zsZ4z2 zT^CXnGdIV&WD?or47j+lLE@UQp~2_r8A6r^MLr3;R_m?x8JZO4?Lf;f?Qp?dIlmn__?T!wRKafc)&oBzC=m;bj#Si zJA%8}g$pSPO@;G!?-vOVyb{LBeUqYHEh5fO4+W6Dtou?|dX&PxfegJP04|=iB3p-U%Br zub|#~%B;OuZ7(La+UNy`#s({?DS8V?^dlgTaYJQ3im)s8a^uzP`1>&%>c(|c-Y!qb!9rC(I;WJ=- z8LeI;BJMi7>pN?m&1jUA`1&G<2OH%VY!^?vxP!#R;Oqo)u+GlTN7&e^e9e7{d?(rM z2Kcq=C5Bn;53xvn|NQwA*RJDng)D0^ySctX9j}0@SO{O|fD0;0nmn*yerEFf8oa2E zRCLL5rr>}8QAI^Oc6Rnc1d9Hsgwu)f)vNcAgM)(@9@~s+Ar=;F%i069p`oEd+cnuE z4Rmz$s3#3=9U}3cQXq0?0v?&CXnB^z@+*qo9OJcwf|q? zq${AJqZ4#neUhoEsUJOhR0u!i;B~5|sLdf%R2zSD`(!e(p)T~tkF%>FL`0dpuT~Rw zXKI$f3={)+VSaB9o1C0{+J4F$vp%>q*BhHWKj_PsyCs**L*~&@ZqL(K{(hkc0W?rz z@@zQ`k+uZe^FJmi=dP}<=1?jLc`dELNCs8TQ^BI5qHI&SAOIp_XilBs#3_dc&rW$N z1{e$E#Nd{UaUp4C;o;%$ZTYykPO>d3+WORM?3qMFM3lZ*gIWB`55LG{m?ofAN7pWs z1seS~MZE@RrKYAT6@4kSk&>2HR%rgu(2@e7m2E0py`GjMYtcO(%3p3V!wdMZ8v7+L zuVZDZC70#j$%bnbP#;_JihxX=t(+A7W!; zgWysU;IvtWy^DK|4$zKr!7F$N4vOO^b!_mT*(lLUHJv+LMzk{e6qLpGSWN;g^d-e7 zCJ>(qn4V`Kb!-b4aXYJ<({n-JZ*3jfd33(Mxrsal@!vLPUIF}C+UomXpSX+|Uy4A zGnJUw2*6tA%PYHI?##H|e(RC9?_e}IGdcL>D^C|)>DobU(*U7+Kdmq&HxQkB(Oc_FbU`f@3!LSumy9gkdx$0G>|9;# zu{T>s#P4X;yRftK?d)((llPJab7ulKUp9zTf<63C%TG2Ir{Hrg5wY_2RLz*~@)a|<>JcAj^t5~DUxmQ9S$~|4W#udO>-bz8j#NZzZ%qh|{GP0xOjE!Gv zX=}3?bUn@0sjKeev2ocP$+EJtdShqz`Y{GZXmYadS^{!wGzH+k^@)<7)Gb{8c2hgN zie9OQVC3m1idb22s2aBjJRVGOfOv79K2@^S)R45%mF|}WB?KV4x%G=*?d>@N3o9>Q zzGPMS(-zw7Egk8-9N%ie_nwZPuH_3dmdO0njL*6914fOQpII2l8lrADLwJZz9#Dx+ z0RXe!t2^O5zYR^8wDfz4XlCt69P4h(jlnw>eL-4deO?6a(S9PL0f!R@SFDu32sMjO zC-5dHL9mZksiLK4^WDFfWVAS)`!u1-=x)ZE79?DtA`dPG>* zU66TBFD}OT>k+7InwqUYFL+Dw?a2i^o5K|IM2nnRsP_~YvTka%K$t4yHfgs3cuVLj!R7JcN@^6zkdD7Onpe_4xlVAz=m3fmtkRe$|+m`cT0QE zd}I%T3IsFWSj3lZzlTj*KYeAf{xyGgceB2JrgmXxx5pYvNdg}%wxmFp2%0qPiBUK- zB4*JjHCFKKKic1K>*~>t3t85IVOYz^VScp>EwEWo;MeIh8 z4*7=2!ayAg?Ol+0Y|tpR#>?Ffv|`NNk);6?L;-hoc7(Xb-&P zrDT6d1AP^0od!zi;&N4hhR;VNbA<8-o!>fQXpZ`eA-k#q%wI9TpzOuUzoK`(gHrf0 zB?OOr+_UIWRJQYYPJTy1!Y79;8o|fHWfU}sJ3(}NIi_9#twgo!84=niOkJ&WkVYn4 zZMJIFHMIUvYv=}^0ft{kIV(l1UqCoj>K8Yjpb?P*j}kfL#1%z`W%!yT#-9(uY(dGl z&yx2+gq^C&*pN)f=SvBg#clQaMn&{TtYm?^emdIx@%#?Z+i8e}A4c5i3T;vev%Ze4 zBuTR9i*d76D_J|n-hb<&N(&0y1P&NjSU`r>hbms=q5HV%rz01PeGmQSfucIok~Ka) zo{wsUEVLbX|Mf#gv@Qb0VR-b6hiesyfU5v2vh+aHKk&6{`4Dy2kTc~JfFFkJe(%xd zg01)K6v#7HKI|2hI2vl{mT~75rRI#5Evb-;m48O>?0NfDhgsW@Axm$KIcf=2aH8kf zA4D|PY!HQJ*HA2mkE<>O#uhG_P)zi4jf%sp^ZXuScd*3aPYuE{ceMjgrWo~*VaM0( z9)4IX3JlBgK$S<)k?dyH3daEpt8xB2R-Q8^u|ybkoGhVj4mPD}yBGWp`)H_J@%=OL zX{G$~=!Qv*3G&mdqJ=#7Z<}YZtO+w}Ty;JgC`M*d@GpxR29R}s{nR|%A1|YLw7ovK zIY#_+`A2!6(HU3hk-kXrCtr)%_4$QnFC}sNd+vI@7Iw#Wtvw_zWgBH_^p?ih`KA(Z z49298u<&qPGO~PouSCFel!Lol5u>^4n+KooJ){&6s9P;+ZWg&Mac&pK%6xqmv$Ydp zDHUlcJ+oMa}*CCdYELYH7-D!r-bbQ~PYWv(Kkq6?dw z{&ep%J{)bLkbR+3?|;fFQ#TF{4rVJyBqs-hd(HchyEpy_7R>GoUvpR=sP1B9veK^U zR!JjOd_d+A=qJyh9@88`2AM}VGJ)x*&J{qDR$I$||AF?I3hBQw5zq@#rK6c0mqR63 zLB;1k9@l@#%xrN>GdNsNt&mTjK8;z6Hc1h5*V#AULyXPXf2B@$)YqoT-y43w<$V#m zjhxX=P@3RgqxyS;TeMVg4ew~{qhn!FU6LI7T7leBDFr0EiR~pW2}wX{>1!kmZ}=XV zEWh_6?$MH5LUx)+wvLxTvAm>f`=g!59#s0O09Gf6t4{m>Ovs-U);g|i0!eukXINS> z%P`1tKL=7l0C`M!f6PTrw>T}Et(|PEolCGjE1ti&{za0 zJBqE3)z#HsGBJsZh}@-l7b6eITAk~e4ay%TDfu}#82j?-YA{ce3l&-6u&0BZtwwGQ z6zB>`d(RpE$fmKj5Ofq-f-%ZHCWhA!;8z^8b#F;;R0M?IQv`q-UY4L)?yN>K*E2zk zLCE}TyhvZUb*9v080X)4`hN$!0IRXAKcgXYoWB#Y6K*a_g5DTrVYYs-?fXmwA<{u5RJQ?*Qg`P;iwdYO0*7 zdGy~nkf&LZ`gPE=QK-iRl1Ns}7?LCh+g)xyxZJw9S1C|T^49BAW82L@Ff13!ocD^d zm^n`thC#q6-+k60t{inp%xm|u{`%bXeVbuP+NGwwq=5Xuz7&4>d|AOsO70?F67kqr z>K+MXsB^zQ5+-z8@`nKX;M?DCrL;ZCOfSX-(CUH7HU>HW#Mp5>>3m0 znf2of?sQe1_?(NQl>}^Wo`B@a%+E|c2VW8C5Z*%^cC>cf>3!TVQws(&d@Kedx@&mf3a2}sT8?k}sVJMz^Pm{aomq-ybRihn^ZUu_^ z5OG!jYCEb+&$B6}-oUzH_7;g;mSkY;S)8L(OZU&5zq|O@bB2e&r+mg5CVockRpsfb z%IF`MtbG9)TWc(+qpOscqEjS`oq&?Bwrh0Jrh^-gSZH%P!Kw+&tT?|kw`uLzNn7lw`FUlIqo@6)mEGj~;Utvpjndhg} ze&0^N`)eCwu%IZ%1>&tAvW{ymSI{H6ULOLw$24a`3~T6ch28nVFA(TKKu9hOa$D1J)Me zeCn)rbblhxM{enGM*_!1`{kwEqzOd!ThNa7U`0IPSQ%9@C77V(?xz%Q1BpHd3!cNS z)XNEOPM3mZh~u?GEcng{boIV2!PtE^zp^;w$efThq(qM{Xo+h`7C-ySs3#_NJ_d#y z>~SgDE35(3-bovN4vuI2LN~4@{nE0sDl3b{RJlF+*>cP_C5F9E}&C9#UUj0NlSym;qVWrY~@WO^WSQQh8etzOH0a5PUg3Ibh-JI zA&RekRHZIIvFsa_SF$%Pc;aTrLk3&)TW~@k2pP&G+*1a@0G^WyWP7EOa>DgK`YFX{sg5zaD+;udYa;v<?u zJ;!1ts-@CH45zu;8BD;Ew69PQq_7)}gb=vXiwU~&YXvut_kRTh5fv2zpdf^XgnUBB zry1dqu#iS2l}0EkDvo@~yS}~#wLV`x4tC8*a_@L=t|5J4D=fK!&lzcA`Rp)sm1f7= zCjftSdPrhRK5%huC1{{;r|9bi)z%Fs(Leu?2HkoAa5s{~O_2syHGr68e|(fx8P93{ zoRW&F2;PZzbQ(zfN=eUay9nBfUF>Lj6h+V;1bLrFsQ|=lAvQVa#JH6BbA zR~3+>KTLCMR{8dLC%26k2vT~+!9f~6w$G|!x}j?7+KZv4U&^q4il-UFl@UR4Td0N_ z!O=;o)SQjOoJvSG+19`o_YmzyRR^oqKr-1F;V!c*m;;oG<1hZ4Pv8w2yo2#lXnz1$ zZPP|jQ9&|R;|&rPjCI=j#sf@uw%9Ec$o?Smhj?h9kWl2#P_bOjPQ}spv{KR=4Ud1g zYJWpy6KXQ!>w+{yN6Ucj79q1OAP&!o39%DW(_ z`NPy%7c;1eOmMj$9D0|}nJCB}WNcXbxX2LkM!Abew;sxTb=7PZ!5Qm%iOSb^+=;In zci2vl^0t(N1jSQb=Gy0G(hc3cvhQc(2wLtqgNaqQ5QO!?^jcbfOOM-+yDq`PeCy{_ zKk*To{|by0N}0C$taI8;0ZTwaEH=haG~uBk7B$1v1|9tFvRm@LnRmcg16>26U0=-D zQ{@~>u~VxL?*O_&!Eq6>+3f5z^k!NhjV}RN&Fe)cuKI1TlUvb|>#Cqd0&WRjXu6rPU~ccf#i9#;0an zaDq$=Y0aHDXtB!i;nBE1(81mX$q2btqY6TisYE3z=B6;KSt-7+OEy3eX#vnlRp{29?HAxN80sp@F-oVu-q}Ty&`2?IY&wv&6PhNQ%*zVz;n7Rla6K5Z?3>N)bp z;@*juIhntf7e1dhu<1QoALz4b(!eF=3R4MFA_2lAOc%Uk`f63$+M^cXOS|>g`It4; zQ@a4T*Va*B>|L^8w5G3Anp!#Z)bl)PN}Mvq$wLBK&r(hZMM^=#`jrfHA{xzY-nQ<G zj>5Bm@({)M@0GhH%BdWX$lESvq4Uuu?^?%B*8872Nw4rA-vILj?Z%vT8(Z6hm1y0? zqIR+p_;_6CZ_C($>W{XZ7S2ur*XQEi)iLg`^`Az`kTk(Zq!6TZzf_-b!U7U}|1<$^ z2qbi2VPTGn^hoFlu}fb9k1ea^EI&6dC&b>>mH)p~Z2W7qDC)16#eLJss(Xqk8wN7S zXRmQ%RHB4A6`-Po+Hd-@4sV-s%8Zi+V*?d~=gP9*62Vt^4&Z>^c%hzJgS%V!9zF?4 z4i<%Er$(n*`=2clrMuNAZ$bf-Nbl@>b=tun;XZAHIH85c7?K)%JUl#LP#DZFUNmNc zN165opoO%lsVO^WZ_2Wv74*@Ki^j`dZ1r53xC%`ZOnmF?_|<&Lzc4i$c>w~!R~$d= zDI2Wl|1MYG{huSkr@EtARGHEvBA%X}o;J?Sk%OLes+8=+#KfkPep_HuS0n=yvET&G z1h$D-VS&b3{OwE9w(slfQ>(P1XbXbm(C=o*#CwAVAftM*n1CR#A^@T61sj{fGa*Mu z#}7clih2^d?Hfo%y~mp!r5Oe>@Plb$Q2>1#+i-7~7po+cEe$Z%yp|V`E^Zq%HcPX{ zoD-~OftI^vSUxbW(n<^OlSeCvId}uS+dkQzdApoy2wlSJIobprv4sEQdVZiQm$nn& zO_besIeFE;TvGN)P!2uE2S*sTCW|7)(l}_K#iqE|v->sL zvwa7HpTJX4p0xg`{2H+|2>7iFRZe@Tnz=+8-R;iRz1qQ9l;098FCxnAu{gDXhL14k zCv?B^wd?}rzoX5O&23y`1RX%rJXFrSlSJxv(qB`vlAZPd!z{F&>ewGoS2c*1aN zy5o5R9}eg?_36B9p9$dnIDCz-;UlgRi0=1D^;kk`g*$BUy4IQYv9#2|DvQs2Nu}uk zY6|jxPDWg0Q!h`HilNm@49?0r@Nh>fp>X9zduZT6>oJwup{=0d9I1E5KjeTd%r`WU zeH*OfTZ)ylpC{Kf{Jw}cM*Sm74;Tlm6qaBRbrYlebivW1gFsP=u@)_eyQrJd-rsF$ zVM8j(UAl-8km(-a`?Exu649xXfNniKJGn{!Up2aHk>J%_vNgzR1+rvmk43=z z@n|R!y5^)P&X`|-QH-HOmwv;jj0RJS?+(^@8W33I(w!3)J8UP zzE7N_NZ{wd7lp90V#j~CnJXu8f#0DDD3NoG|Jh|mUETUbd~vB*jR#(ob#L4D3m9z- zbhdSJhsBpr0Vz!+tb4AZqw%Ha;&mj&!t1p*E1^q=RpX;rS$ES{cn-_)La<^4=v5V) zJ85Z=0w+gEXy|9<9F?r(vQGV5?cO&yxV}$mBVrW14uUuqnehj9idbtuH+`_ok{yPw~EsO7lkJE z1yv{EDS63+6eybBYt(y9G4<}14vrskbXsOw(GT}X0_SFj9ww))Ei_eB%s&l~$Rq;- zmv`LektUB!;?nYRTJ;u4w#ufapv!_;FK1@P2wIOLgXusJ5tBU@k&4w$goHMq!K9p8 zsZpsWYnxtG_zz37R7H~bE}~d#t#5G5`k9)w12~_D&8XwQl^*Q_ZO(t^I_e_80!mL4 zupqqiZ6uaTK)lO<&Mi~Z6%Fkcd*lgyr(f2o%n`b*ossz+hmZE)1qJ7GYET_r4rt6e z;4Em`hJ#*DpFK+a~3j7Up39h$cRy^5(hYKfWh@Xja*W$RAn@ydMXw!KK>7Xvb0mZ^FJGEwT?yA zx{paLsyAqqA2a*Ow1wCO1xA> zyvUYA+kg1LzHsUcCIsRAdPbQN z2?MReH4O+`+(t$Lo>D1I_~aH6Z+z?>kAvI^-1dA@U9t((XEzQjO$TYr1k4(=Gmiby z{W9+q3xx$5UD|cjGyAj9>^7x@Q#-EA3dh;SYtQ|}5Ij3F7G*%;p0xhVG-vK`2RVl0 zgIa%(m5p6t&Q)mIrL%Y!#k(3BF&UguDK%9V*W)cy)|UehZz>QB1i8J-Hlp^x%A9#; z2Q$wTbTqdO1r-O+*e|b_k_Q3KxV%2L%4-p(mE+2FHG|S%Z3K;t?Y3A7jeBDlXjycO z0)HHbhz7dQ8bNk8pXNygi~YjsM1Z6D?++Y2YS{21%?aBLpt_%$P2uwI9}^^SJ1A@w zb_B+R_&%rjifQmH5>|20bW~suQQqoqof*gQ`~v)gvXJ6w`DvNDqSMbQh}3*WuF^fC zXp&;_;+K)30Uh`g$qNE}**7oBK>?8z4Oj{WTw+S}((ql(=)!Ay!%sqjuC7m+qjZ}E z0=Tk%TNMkKfk8HR&pfy?Ankm+MkSfjm@GKs2Ye68s)>yBIYvXynh*!fyKVye9Yact za|wA!82sb;H6bjmD3qdpJnFr1_`S_!^KQ}K`9B;N%LNfs0;Vqxe+VYAN#(8YN26-? zr#BN3=?{xBL0OXtIfJEpW6X}Wd#01f;T~!*_~4AZu_=S&|H_Q?Pj1q*sSv$ab+;2V!Hv*rIy?J zAI@E&lqixe9)qeC@8wq!e!~XdEDXgsd8OYCA~Dn_JZl!p$y8aPh%a({gntfC%Cnli zPs&N#ynFB+ulr#49V`lv+~zhnzGUkQTc2YYJep6rIKLbHJg$Vp2#Spz)6bpG`hS-c zZ~o+ERu&2}?2rESeyysvXAtRN{p;o%`g}RP*B&1UqNGAc!y^KuTly8Q?Bw&b+BHs4PSd#Gl044J z;vDluk8BI3qwUw9>RcmO1SR`9pjX#K=Dxv!EnZ?HPk;wtw~f)PpRU;#M;Gv3j7NXn_jo;81kGokz0F~Kwk_dgz_>EK!?IgeW|J}t?x*J!WBQ{pv$t(Zu7>kE z`uA&SUNR;BYaza>{L34JA^tWsgjAb(>A}Q@eR2$>Lr6rhmFz*D<~oU{*QmPwo-uJ zIWaM@S{WkpTs@vYDmFGN==9XFCxPev49iqYgFKN#JhGsNT#KHYSwIb+t>IL_b-N#i$$(iMYO;kS+VpgaU4* zh`6|H4Z)O@6s!M*J!Id!22G!rjErLL?t-8^|OY6P}Nibw4}pV-F8N*W=YlhPOeaCbOUXvXuy5rKwG@iwp^ruGd^4NPMO) z8~}imYptP*V200mzJlDW{pv=x>3T_vRYmPJ1;-zm@9rnUwrdG|4p_jo zg}vIJjBRPThc+isef}?^a+kRoW=?x?=ym3#f#ME(HyW|iQA706qt6N#dKJQfCzQzZ z{N{D3hQnI&Et%d2;z-T-W*XhCvu84N(F*}oGGP5-Jx?euW{(WKZJ@kM;4T_G@%#2o zTtg#{x7u#w8F70lmdq<~n_GvQ+W*1fCZ$+AzqItDEqM9(wg=Z1l;pUGF6}Lo2GbKQ zS|Hi(qcSSRN>?u~{>4X9vj4#jTWiiox9iWlwj(%IGxo3`rShzE|WIW z6dPXw3sUN0`d58c_A~gs-1%=S91a^BTlTuJKY4~+a!_jkVa^2?;8@QXROWBl)qImD zDl{g>xO2a=PdS)s!wob{ejYDNOrgSf|5ab%X`$XM{JMSBeftihs(JmidFicNCz=KZ zMy5Po$B%OOt&EdLtz*6gQb~?MLg@mXkOI!nng2+W69dii+j$EuU&#KFURjE^QsGfP zqh9~-$Nfuty4%gS{nDcDwY|*(jrFM1R%D5#{B3PUpsSBDGhLKg7)^JFapWjsQyo{! z(@-&t)`{|!wYKpAx7jde#;>3Ae%R$H^V{NiqOTZ0Z6>%Cu_gjmItCWF74;R&Bgdl8S=^9r2x;JZ4 zu?$khs3}^Wl9+>^L`o8rHZexU-YAjxo7MW@?RE##5bx`7eGy8;XA8mIN4v|Py~u4x zKG`k?xuP`aIZNY8|E%<%ow{D&mSG3?SW`?5Pxa&P$&vd#c#lR^ARqPOJbAKtCd$5h zCZss2M!~+&hJ^?lMAP+EX+Nqp7yoV*}6$3)09Pg_dyBa{8mQDxm~(OKieu~(y&_bj1ezZ4suZv<}tb182pH%&Hy2@q3T^WO1NiMf;as3NrO&rUYdk*OJw zMZGpCgQQYeVVso8E;GPOR*eiNSc+Szp>A6rPk;ZObn~>{+kbKCbR-MV5ae{>Bynd;DzE)jzS$Th4mJDPc?$CKpKc9)+@P)5pUUh`hCDc|Y$g!I zsIovbt>O>);nQkdX-UFOOiXA3+nf(dG6Rw&v@iMd=ZyxPw~ML%V+jUA#LO&rlwuuC zD4_b(X{5D(>9IRO-Wtv%K*^1_C|Ouc%sv;yk>>5{3+!9B^X(c8OxXYAc1A#gGHh&Y ztcGQ{C@3lKhK7c!6@K_UN@0=@3<{#&PnsYO(;#%?7#JCEZR7)>O>txiKV0qA@JY)8 zYa_I;gf`|HgmxQl+_fsLgttO+ALAZiZESq{eagF!cQ6=+?bg3rD^1d~u(Shb9I7AKoB$IuhJ-2WiDXMKm;aT=?%lh$bADGVo{_c~xDRgq z0$cL0fn$QSC!PyCDCn}lc7I;DOstP8m_N0em0FUJkkAwoWq9k!r%iUn6L9_`ueS{G*USR@>LRtNTKcr00z)(=#-_#`R3H!4$U1eLjmgK_faqa5p;sO8#4e*)V6A}`7 zLc;lOOWsac;|@S2x4zf&)S-Geml`r#0k`wgO_|Pr`A3y(bgFF9NA=X7(WE)2i9_$5 zq0$1{X5(jW*F6vm>^I(v;kHg4T`)FWYWsF;X~QKY&8@a+xH{J4Kb`l~@JS~=YoL@s z2B`a!K!qoA05_zBLdD9SYintZ+(gR6bAoZ-z7kq``p362f~KsAN!Ig-1XjH-qk1N9 zH3He+zR;JD<3I2MH`#0@)K$`zPuo(w)&?z`w)}=W4qKx#NXxyM8r2z{3Jb{j(biA! z+C%es=`KTysLnj7l6?dSol3oX3yx_lJxCmlq z2nf@-fcYU14rzq{6$SsB^+>#B?pMzFFJ7T` zG{^m%6Ou6WOw7D87@Pr6rpY(Qe&_R^*dzXcD1b)Y{;bPh)>jVdqK-)mu4$`#*T)qN zS%<)@>>sV;j4pI@S)y^3dhRaCS&SXMO$Xgd?nDfV5HLkPFgL)aSJ&2@_32Qt-guG9 z8FJL+e!f9lhL##qX9X9xnn+r_U0-t2c{jIsz)(RUbW%?lFT(M3!ROZ6xVSx0f(XDj zVMBLtIEfpJj*4ddK%e%2W@}yBWv49a@|ho_H{dgvX&5qi0gSa8Q-}RRx@-3DHDg|* zFxOUBRSu0z+&ZJM#mfD)?ZwPrEQykPT(Sj`3*}oR^CBPnU_qMGrK3l#7J(g8HShUE zVIB|+fGq4;txJ#>IU2UjLY?=$U-07HjK{&Ec-hC-p>=v-%@!uS=IgDWVCM Date: Sat, 1 Jul 2023 23:28:43 +0800 Subject: [PATCH 033/337] Delete visualization.png --- picture/visualization.png | Bin 11861 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 picture/visualization.png diff --git a/picture/visualization.png b/picture/visualization.png deleted file mode 100644 index 399bdab910df808707af9799cdf7180097990074..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 11861 zcmd6NXIK>9ldcMgN*2j^KvBYwqXdB=iGV~UjxfX_=L||1hRn#13?eya$(bQX0|Sx; zBxlL__W0l3yU*Qy_PO`l{QxxGr_ZUbbE@jCw<_$lnj+By+6T98-6B#}f@t2lbvqIG zd3EnDa0fqQf8o|G=3B}T8EwzB?b-W=V{X^ddsHHUfe*4kp3DR5s6a1Quiul6*5*m( zsTA-no~xR+8toAeLd^RYb3NR)uNE0xHK#`WD0JbW6k*V3CAzIxxfLI8?S(O z3emUkp7PI&8ISfZT~J%BT%@1z4x9*O9Bh8f$beFklHLM=K&AB?M!;WZ2|_9m=sP_+ z1c?;vbz}j7hWQMUNF|s5>2{iG$lyYV+^@yOJBjOs!zWJ)e+xEW*$jHPB8_~;q zvV?qT?@Rrz8R%!T&y0PCnx1lU3fLyU<*h%PQ*gn^I`ws;htI{2@ccGh6FJJYHg8by z-8X*HKQ5^2SO}czqnC^f5uYpD#L4txR56B~mUcSq*QM2+N8oTf6#3TAM{H%BoFqL% zcJwSnPg|KL(lG4*z<6Tafat<;-YB%k#WH21GJ3 zla=wGprKuM)!(xJ5G1z}{^PJLZ_!RVDWONvpqorEK)9O7kM8fgZJt1*_Q|~B3_9W~ zMzngsPiXwkQ+IGe?@a=-!<>NMW`2Hg!Z`8mUA6>_|Ch6gQA6JPPfIzG(dPjl`<#+W z8u~`kxZ+C zIsQ~mkLA%kmJ4m~Hrg5#8u~7Nfr({*?j`h$sJueF+p*e?qP5}>r((^m=htji>DR2q zB)I272f?TAB>j_nilcAQ!*g;v*F*v*4sq@pV^h4|r=-WiEuN6Ep&>*>rPs?8)g1G# zvB<1ikK#|fIW*bL+u2!@rWGuaMR&hud$xVw-;7P-uia1{>-3MctM>Wadreg&m?<$u zxXo&jYC~vV_K^3xuXaON|BNN9U+SoT#^K&+2d|r4p#>*;wdKXNU9Cw#UOcq-G9p`o zJd($lQNL1BpxAf4;M)A%P&MKt;YOMwW^h$X*qoQLS(>0`*Z0;81u#|5(*s?bPFGC4=>Z8@4CA95m@Z%Nh33ppRH;a?m&H^-h)ln>8Y zoThVxzF(lb5=W=z4mC`UL$A#2q90#qR}{1<-^uLdcr_sCJGaDjZk}@D#*w4OL?9=L z7uTc1?KAKbQ$O*j2*E>KFrM*HiohT$hORST;_L8O8VBZjgMbNCMMN8{0xJ6QrGQ)k zLk>AY$-}g0&vYv#Q*UxUHtMH8wZWB*Su2{~>y@x6Ft>HKEwB6aU}J}m+!9x(zxnqQ~$sTNL7vt73B+tKSdSIZ_vHw{C?=h@ywy; zkKFQt??-SoqWq9FjoV#alRVGfjQgTfjOa*uq5j{#cP9&XEeIFujBP!o=LeO(id7V1|naE zAZA8xBQ9hVE=KRBIEG~kJiv0pOZYr}Tp5F}t+Qm$X6`0UZhUgdT(J^#?3BCO)mg78 zj&Ew6upW}aoRn^#{TV%9*-W~r;gn(@+iw^<5XS`U>fdk=?Pwhx72RA51&9cgoA1om z*WLs`%&HC94DZ~zBS^#rcu;%cno9z}@!drLzpbMeWFe3gSUo8jYK0mMf#it&|93m5 zl19Md*K#7%eyq}C){Sb;XIC;x1!Cs6<4yIxFTr(tN+*3rF_QY*+Just+IR&7avUZ^ zm38;&vuEF$?m^r?EXZmvpcEN9_LqA1i=kH!xD9H%ipyKWvlU}HySwossCX1sN7DIh zfAZGs;*dzZCuR_DpPy&)m~p~C-k!GfPM^PA5%F9Ip&A=a6}FRAQo3gxaOH-JhnETK zjec8Y7AC*k;IwpiWhglaY}8l)%U4Se2PQ3AEi-CDgfjV4-@bj@YPL~macRlv{LuPn zM*4c6Bse4_Q>R#WZM>8&fd^W)+zT-yLl9_$m=7it-a4CTyAI)eQ}*km?WSeDVK`M} zsr>`a>7PL!%oQK3hgm>?+;SvMrp^C~*L>g~edx(pp%%v@XBQV8rN@sSFI=7-93?2q z%GQ4QB1kXh42fT_Yr87--nlxNe^Z{Eo*q?gIa-ugPsy$S`Sa&n>e5$&np#>;XMfCX zr)!D)E{^RM+2zI@ftX5gc{Un=C@(J;{}5sQBd;*!LtXC3Pt1$^empvq1U)=VbainOnvszq zK7zqe_?+x?V~XGCs6Ko0M4^c!uum4dWH17e;t%HL?sgfK4BeS+Dkg6vyhD0+c4iS1 zpU3HOk_xl*_4U=}&XHnQOM9(qs_>^8H$pH#nS8E5BRe)W_WiG4?{o$^z+m0m_0G#L zb>dL(XhiJk`T6;2&kV?g7||gG57(iEyZa>4m!vl1B^^CcbUJqO=jISSz2rP|)K^jW zt)%tVLwseW8i#pfjN$cRzO>2B^;OZj&GuBy+eX*IDRo&(G=fOf>kk`kb{p^uzia0{~k+;T^99y`0yc5*&YJX zmQHTcna+8qBIaAKRFRXPziC7W0yU9QQBC}wcvnsRf9q2$3^~E_0}0T*-Jv{H0N^?6 z5>g3B%o-9cIOB5us6GNeAVV3M)+a4&Yy@VK$^3NPBMet@STmvr7DGE0cJCD&*ZY>@ zO4Cc`duSk$Tg2#G%DR(#+QTp?lW6vT+-f!6p&pZLCk5wg=<4QqH~}LcpmT{?vjQCt zZCU*LIlneNg+Sax7BoXMQ8F94a2^Q6ObnsIAUk!+TqVcpo8IA11|ww0$l; zwU3l7{2u&=@50UT|nAY_8c$=o7TnSIqll zr?j>s2p@(uwbe%V?Qr83?@WjJR%Ap(wSKY#0Z&G08>;#!49T@PuV87V>VW4eMyS}^gJtF9R8Mk`K`|JM{l#u-0gK4>)-h^@X!@}0s|(i0 zk8{VDev4TP$8qzhr?PaRv!;p-M#kAeMHxD&-9y~B^bE-307I_7yqLccmaKK#)WyZe zUm8jlsK3~0BBP-Rd%jbj-BbDfpC_qbiyZQ_HYt%CYx%m$pZqO7RE^`1eHwZ-Df8TC>akIH;GYXhwjm%;QQoH3M_$!Zug|7jk=Z>+>_irP9v&WC`d{v4D@19s_U5UkB9fA*+1S~OS1pH< zJ|r4@-?tq7)UBm1X)gXmQj#HGJwvyKLPJwCFqF~z9uSpkE%v`P#kHlB@~Fbd?eAA^ z_>hxLl4C8kYCu&zTH)M~E$z#n3w3u^x7}x3mrJvIj!X2~Kz5{`CgSC8%lucVOz7MX z_Pb>F)II+GO91@qD>2JgZ(7^hkcojnx*V+^S<`Z6lm^J}W|5LjvLRGh31bK~>uhOR zLHscb-|uyl8Zb9zGqsuk@F2O{(K=@VyV=^Dz(DLt*4U;7N3m5OP1pP}Z6hE<(GVso zIJ>!ZR3Mpc#gam^KGJzpg(Ca%cnwotoEw3mJRKy8Vg?z=0)B)F`HU`~!Htb@6d9RL zhokkK^8vP)eC;({VVW?h3}BR>ye{#!y>DX+N2!y{|ZWohydIb-a;>I_3w0d z(DFDvljqj>lY$BmHfk{Knhr4cCLw@iQ19xkURZ3Z^h#N7l9R>5>aEP)vSBZ2_zVS2 z(*P5~ZW%RLpKo|v+7SUr(6^UL=A%0%> z;uSA;7GUTT)xQ=n61|Ycm%Q6Kc@Fu#1&AOpJif}&5Bc3y!U*1+C|86)a68dxK8Lwx z+EE^8D}9U4iS9HIFe`md>?2}o@BsN!Nw7*4;{i;`+WOhYj~`=GQo@07xs)9vrCAs6 zwmD9eD&o-dNz!MpSNfVPEG&$Y$%Zp$etF>N;Z`5kn>efK5+yP`cQ4RaeEXJKLQ)b1 zL_;B)aqUS(I49r$G^9!BMM=fP#IP^g6OaTaHWfEs@{yqgs^Im%S5`oRf@7}IY-|uo z-PG;~D%emGACZ*sj)Oa`vvTBy1GPEF6{b%->AOG%grrx z^jeO8L+e{2n46oQUtDU`C3Zh8%>e_OQgC;#U2hfi!o$a(_^Ezw@X2%W zK4y!8l=SOom|aN8M=oJ!teRN>CHVSE@Cyi_;^RZIvOpzoEAK4z#*|l$0w9jhX%WI@ zP)jK3b8P#yvKD)A`jQ(Aj)rGuE3LkPB58$QMqQ@}+tHeun#QK3m3+$JgWWauUVoe> z=|ib5dGyaZwe?)l_FID|**%ngabYfa@>ZLhh1lp&((_nax^3v^;csM)7PM2D#K#%@ z?lP;?6Q;@ZYKtmMg$9YFeHn4^#Bxm}`{%m=+`oIqf>06AtUk&X1x|Mw2|72KDCRd- zK~jW?7Q#8Ewbu&KfK9P?I$B%gO`g-_V~=gxU3B+@ocYHWGdgC@*p0uBh@envi}+Q* z2Xdw)q~curwHx(Q?e7T;S}=XA=`-nd*_;9#$lq(!U|oJtk)-`Ux2?7s*F)oN zQCJ;f-+KjS2F~KY))406SDJNZQ2{xd0rE5fQeS}~zw4>N;(&!p`z$xk$+LjqfUFFC zwtnlQ^RB$=SOXOzKtvGd=#{iE0XBkKGOb^02T(Q|H0^@@OfE!(8eqMD^@hDRt8J~r z#}0%61G}ctL~@PzMQ32_cSj=@vRIQ#GhD7Yh|^cF3grUU>VJ3{2v}|3H(o>6vAI#Y z7V)#{`4A*MFUr(Iy!g*@wM7$Dve_@}2`48X9Ur zE7IoXX5N77OO8>u?J0^D?<3jyRzCpASeAlBij#hYPo5h-PDaurn}p^|zI=J4tgQUA z!URWFR+f~BDF!%)tm5KRU(}~-?e@nF988WjFcwf&Zf+s~DvfhER||}c-(2P!8uB~~ zNo-1;lLvv|{Keyj?pr@I;anyrCVPj65Lwwf<0X1p<{6(qGgF8IczT>GrOd3>Uq<7Z zyIJ}VEH+88a;@hwtS3zb8)=f?VhXr)I^3Kff%+Zkr1$>2%|hWkf0A;48(jCHSiiLe z1mxu8j5TIRun${QEiX^{L+@hsn{Dk{e|fbVfQ8==9*KS6(W}B8l{_&_?@dY#1Aj0n zEH$Vz1@a&zRn_k2yUjsBmXtKh!^cOqzP=8(nJ7!|E-$A_E)6gc{kyGzSk&r)o8zTu zSd1h>%I|#4$Mm0+mstDuS5qSn3<~1*S6^!V?ka53iHjLpd5Sd-dgW}id`ezU*vn4y zu>uVskn$TEZTgF?tMm^IL3DK~!^6WXOuHXoG9AJK1MeI$_GIMtJffqU{<%1vwt!l$ zJ`(xsAKoj(P#QFQ^7)^yvtz_|mzIKDR!0i+D#hoAW*H(VxiwAL{%hRT%2z9FeYie~ zs?e`^cl&sEKIZb$SI2-1$e29WGDCU257&Aq=T~sgys|xzZL}ZL1O*Lu7lHE(2)*32 z+p2Y->g(sX-+|9m&>E{(ZGrKb0TLgKZgOzAMkq0Zwu_$I_qJ*vsp+Ad029X8#^6Dj zWUu2l@7b>BVtos3>9=810m~6>Axg1@w)2hRwePtEd1~oJ=96jBTzIUrx~W(lZ5d*9 zRBx7(IHZN5jj-*JB7@T>m(F*LreckBM@ZaIzXl%C;Ie8uS!GTwVxP7l_Y4dkud-EB z93B@LGp?l{&;mvxw{)`6=F0=|@GG9=UC%M1)Kt{-p?7i3u9aDiEKM7>Dn zH-7*&ZbsJFOmt39r}}|usb^*ZSyW*Vvo=}=aItP_=lk0@d;7gi#TgDQN4ma$`q}jp zhM&ccjyr@#;g2{1mOWXjKq7xV2)zzu@;{XsPN1s79YT_^I^JBJ*yxH{K30YCCu>@4 zD!N+zOB*HMgc9jQ?2`iJadB}AdJEPOSaY3hP?#_@b8jB{k|E($)yTteiD6gNT>85} zlS1gAO?e9fNLaN?wYmIosvcH4c~Qc}v0vgdp2Mjz=P~aGXWa&HsfY-ptbPxT1SysN z&20!oIHzL)6vsYn6eJJWNUfl%5UTJgdCSpl>U69$k?X|y)}}%yaWctRa=B1H*UbW| zqDFTkyc(NKCzo&>ePY~C12V27Jw$G@JWxO~0KDWR^=ap{t>C%de#);GI`)q8$;_P_N=7B&F=HUnSl}V|?ij9FDUbw#4B8zl$se-0>_o{MK-HVC zhD{6`bt6TXwgxrKAZBsYV0Q?D{l?xbxTs_3Z1ENts-pM@eYLRJ9iRZJ;@)zplvAH! zM4XWvcGN;gOb2|XomxLD?SlRSuu83y6OxV_we`>|b{>=OP4f$5-8SA-VaUGV1=C)I z9@qAZ7@~G%xr&*Sw&~b-=x6NbJVHdI(~U{qVv-aieG{Rw|KgAE)^jHf?38?l^fKk7 z0lpG2QXr>8?d_cWM7gN}Pf&5NdQs%gw*F#VauC_?)euEWhh7ny=J3a4>!AyaF9Hqf z+KbkMNWW!Ld@?-G*4IczDv=OU;gf&;PQWB3MvBu|+r1Spy})EpeF%OC00l$@4t#w% z`f*G*%*@s2@(g-W`EH4&Oi=-%YkuUY*X9ps4n)ex80~eig7K7MWyJ})OB9%xNVPJW zG5;C1{=|Vqdvv>A~uSYz>=+)HOu`lyHx}LMzUixrzAxuD3PPtTrf8LIcqR<7w zjddjlw|-X2VZ=K-JAbXOhXn`YmgrUI4M@qz$Z&{CNJ!B7a$(puQIgWa4@RTI+cz@Q z-kn!5In@+s@0yaKf~x;FK7+!U|0Ip~=H=utgTZQcUS3|RoL_Ka-z(G7Mz_5}%1?C& zB%PmZ7g5SV#tchn!7JkN%U$_Ll>lB;QdfUt(B!^NtE+@BygUx}^!dYm`t0J)L(Nvj zx#dmle$)S>gO-QWO-@ecT>wDw9x<`~F>F20#~XjCMRCbVutv^8L(ELHnWu+j4$EZ zM#9#-+hsF0lHxD^TW#pGhgeFK>xRlaK{ETWDGAS9**SF1FGt#bw?m2pWSQUX6-UmP zW@u%=RFmawiQfJaaQWU%I%5nl?BBPj!NH$(=iZ9>k)8*C>2Z*r35#*3p8y`2tx|(? zPV4sCt9l)}86>9{NHVS>Z)Oi8uxmf>ab$_3{A91>#YVhzfx8rcM3dCCbBn5(ElZ(t zW6IR*LSoja@>fS|(UXIn1Xe%=!EI`5x#Q1RK~;~?A#g_^MN(;tmZ1ZT_{i-E*oeC$ zFt`<-YYwmhdz(*_AFFIixOk{Hd#VeL?>-OdO}K2Iyo8wXAqX(mQ+Bg^JvmaR`=?%n z>23E%Zw@Te>K%2kXCw+%O^vZJZ6hVzVMKa-Cx&&C^zJKr6F(uj*R3Uo{x&yiu)9cx zA~GEg;riY&PscM)7oj>z-0gSu+~n+dseTC~*xc+c{pQV+ELv*t7<0OQUHeQ}%l`20 zPG|W!F06l?i-O-pi{zJ~3~H{M6-3wJn5nQ%|0@XVQF<(Qx(bqxl_s5W>)QShtps?H zQazyIhX84jtm?z@vd5HcGl4`mUidrVgq>Mm(1=)NbYN7d<2r4*yYlZ6K-*z^pLIP~ zGyUI6U$_(AClL%*d19Y>LWG6qoD(XCU`8@M8gz&t#bgIbf6QU;)J;7RPGe2?WjS%- zO4aY-w$Bwv7P;SCfaF}J&(DenpOvf82F|gR^6?1Cxm~uMY?A?m6Q&7US^~zDoFi&` zE`Hqo^&|POuUYYRdJ!^oF&(|=eP0gIr@?2~$8qjq9kHxV*cIOOVU6}zgtT8PiAPKA z{lPjWIgNgzmIy@%Ev!CPhlr4%ePA7jNXCUz{M=`$?WTbWZ9E^WBW;#l{|_M2Q);l8 zTh-IN$fp;sEAK9sDm}j(9d+$W&jyh>*!EaHcX{m{G!e^E_e+r*}X6&5Y7tL$fx1!Y7mz4y) zT21#n=#Z+{*ADD5mo^1*ou3sqf-isWtD7BoN>Wwn*&||;9aZl6W1drIY!B!p8u~k1 zivtAL!h~NC(Wsm3tBMS`E-Prd(~%{0b!R19*(tNW&Vk)G>&WOngx8JVW}t2a&0*C; z{mEf@JJZ7{EGU;wo7}gEJA(M3Q|;c zGt2xaj+4HyM3!`IK-d3IN-l9B4*(rsyjwkX=YBT0y7HI0Sp#gM z=Wg~viMvtV(W{Eg%c$#mp%QMX8Gpkl!8nRrbIQ*n(+)zssp9(<0E$XeUq3u2ht2k9 z#cdWA7Gz9J@9F*$83Tje6uXcRMRIa7e7e=|xy49Yp?%c%@83zOsjY@h;}||(C!_cn z#@zSeeB)QXni#7b2L<7V@u4mQag$f&(R&UZ*EFC~VE!stFR>2meFB1@{CxI@4oe|T8l%Blch_>4IwUc#pp88?4c zWULdbpfOz1**T7mxa&{vd0`>N=dF^o+Ovx)$(xbotsfPwS7Z;U2}muPS@Gp`$IKKlx{|{y#f!^n4O{T7>68iLP%-g z^o0-2Gma<%;Mw+KnTt51A&b#rdnq#^gXSUQdl^K<`rJq&&YeDcU@$7aACw};8aVbu z4cds2?|GMm+8R%t65iw#3rhdV5cZ z$OlZ4T0l&H{Hiu)LDq~_-@5^3WfnWtbruNn4ftJ>sjI~(Y>iJ+&};tl;B&DW)e8}3=z0R&iLN|+|hean^wu&VY}^27>}ldMLIC?~;`)MA3p=^Dr@c4-Rn zO2pSNVcSwsj;NAgK)g^hHvNeg(3-182lk^RT(yz2VnjuQ)q^w#frI_Y5yKx&MmWBN zAG9>kg$3)ljw3`uI0yn%P;5JD5T{RAyf%A%Um&rpumbfJCG_3h653AXV{iPvJiE=q zYrE&|Am5Dp-oeTrp)od~s)tWycJIMtBo1CzNGE9P%?DV)Lc!MtWVwJE+EXmIw)Ioq zH64sQznbNX=*uWBAj&z*mikFxtOO<_ACx6BFO5Ccic1-Dp1U}{7p^i$o*$q1Jc~mp zUwNmEyGZFz5!D~K-ebSUe4J##7_7Cv9mZ|2cYtCbp%i$!UC*^lg zO!;nqS55wunvuYMRyN9twHFXE>n_Tof^plR(5{nPgsDx}FdXplqA5@$Z1sWDrR|4C zNzV;_VMb9qxG_1SJ@}rr7Nek%&gp5?g?*UVV`jQlncM~ zs;@~Ui`9d%BS$vJ$c5_l*NlI_(DJIbjJl@A5}EitYXg7jirjqo0P7WCU`lVkuk)ne zRLy>GP>ot8hkXiRegR|YwA9U?{T#bEe!c<;3C z6kCbt-U}aKKdY>PJQlh2E5CNb%&{YkU_$$sSw>U13*LlN?Pn$K^JjIIV5nSAzskny z=3|ke*4Cy>-P}99`S7)*&i+?Q{oeN4A^<->$xUPUwZT+S-!JE`>j)|TksZF8k!Bs$ zg9sZx?&O0Xwfc5_p3^t9YYc@Z9n`by?JzT;YWB@UoA2^#Ps-F9CvD9P{ zyope)`Al0EvTWkI@m*-EynulMqxrl@4KCG6VUjJ|JJ_=5FMrExmN1J_Xs0Q;bJN0+ z#$Iy97LJF9EfvscHes2XT1Lt(9vt!q5ifO^DU5k!J}g)kaswAamxfuNv@c9Q(oZwv)}~>|6Sdqi{%cslNO0XJ8x|N#pKibT*=aG2Jh!WteJavb%3#jCc?WbWP%^1NAhjJu3x5H_=Dq(GFtB+3C%_O9{TKwAeur`V z_u%iZq=fVy`sl|c_ODghH4>8bdOx3QPSk9}`YRQ_r#(sS>+5@AXlMuvk7#a|_yRDA z8olDQ{**fV2L<67y{gfBdi0XsU;4{U{-Z$f?*!shenCOO94rc`T4KhvAHD>bNL7CZ z#rDxG4u3}A{ra#Bd9h;Qi02o>=!LHMz940F6Q>R z@o6@IwCP!f{-^%LZo)DX_c+2EMse4tSy@?Qx55D=wqe9J+yDNffqets$&5?Pocy=+ zlFv>6ZN|#auX)9Ew$rqG2>w_$=X;=fE}}n%16M=k)_H`b3%PMk&cH$i>(@vtSDmWZq-0O6PnCUARR?)PH+ z&U~zA8*4^#zrO(xN11U;kv#zcfld0_HX$__<49CJ)&G$cU>Z70>*k6)G+YIMN=-As zcsjZ*XN&*uV?>o$yd$TeADwR~aD|CJE2i zvp46P;Ek`5~2J)5RtwrYCWv>yTxy8B)P-)I9x$o++eO}hl!>0dZTNf&7 z>Wek`8{;R~-11-jv?)p!>m<~?FI4#fRN0u~J6&b2a=GKtrbWJiAYqCIthi7{(BSrsB*WJvjq@wHw4iSRZYcNH+^k$=LOVsH`NM+vyXlow?# z)j(5D&BqV8p{c2VJD(`I|A~#QsoiNlP`ezxxr~_#*cdDH1S*LYmZOX{8z1(a?h`je z#~PB$k7s|>aSP$P_bei0K*-gM!qkfim~()f#-zB)mMybzvJZZ|B#}D( z?EQ+e?jLzV9^>YMvYDqQtc-q#8WMZ$IM@PyV@c4MS#Vul-TG0k0$YUKgF>4AMvJ;x z_f(+(`78x2u>2sXrL5GKl@Q-@`DajPXTog~(U{T*6pdaDHH!}?`K>rZ6{Y-r(BNae|XBOG@b!P)A4svL_(v<`g1is`}D2rjOi32*H zihIkUZDqRSkE<+88i4h<_@pgI=-s+y?mudX?gff4+2Ayw2T97!gzrAZdLkK+UPxIC zC1rMI|KaF-D9BI!tD6d{UA_-|`sSGpnGnqz`6uwZ{Q9pHyvf8kg(JxR)zvrrzJ}7- z9SE2IubtEXxzD=6b@Sm!gtJX9UxwNCL~Pc}trgfJlpOWv=5{)WnJqoiPy=Wezojg% L1}T$$7x=#bNU<4y From c2f5be41625e54e59f43a1e1aa9eb701fe86a835 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Sat, 1 Jul 2023 23:35:21 +0800 Subject: [PATCH 034/337] Update kernel.txt --- Note 7.0 pv documentation/RL/kernel.txt | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/Note 7.0 pv documentation/RL/kernel.txt b/Note 7.0 pv documentation/RL/kernel.txt index 1d6bbb21..6f879b37 100644 --- a/Note 7.0 pv documentation/RL/kernel.txt +++ b/Note 7.0 pv documentation/RL/kernel.txt @@ -1,16 +1,7 @@ Pool network: -Pool network use multithreading parallel and random add episode in pool,which would make data being uncorrelated in pool, +Pool network use multiprocessing parallel and random add episode in pool,which would make data being uncorrelated in pool, then pools would be used parallel training agent. -Each thread have a pool. -Note have two types of pool network. - -First type: -Each thread number would save in list that is abstract. - -Second type: -Each thread number would save in matrix that is abstract. - -It also can be realized by higher dimension,but i haven't realized. +Each process have a pool. Support stop mechanism. Support prioritized replay. From ca5b7375b9153272b69130ce23416b4e6afcad71 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Fri, 7 Jul 2023 13:34:45 +0800 Subject: [PATCH 035/337] Delete Note 7.0 pv documentation directory --- .../DL/neural network/pytorch/nn.py | 53 -- .../DL/neural network/tensorflow/bert.py | 167 ---- .../DL/neural network/tensorflow/cnn.py | 27 - .../neural network/tensorflow/layer/LSTM.py | 37 - .../tensorflow/layer/Transformer.py | 46 -- .../tensorflow/layer/attn_LSTM.py | 44 -- .../DL/neural network/tensorflow/layer/cnn.py | 50 -- .../tensorflow/layer/cnn_lmix.py | 58 -- .../DL/neural network/tensorflow/layer/nn.py | 27 - .../neural network/tensorflow/layer/nn_acc.py | 32 - .../tensorflow/layer/nn_clipping.py | 40 - .../tensorflow/layer/self_LSTM.py | 45 -- .../DL/neural network/tensorflow/lstm.py | 26 - .../DL/neural network/tensorflow/nn.py | 23 - .../DL/neural network/tensorflow/nn_acc.py | 28 - .../DL/neural network/tensorflow/nn_ol.py | 37 - .../neural network/tensorflow/process/LSTM.py | 46 -- .../tensorflow/process/Transformer.py | 55 -- .../tensorflow/process/attn_LSTM.py | 53 -- .../neural network/tensorflow/process/cnn.py | 59 -- .../tensorflow/process/cnn_lmix.py | 67 -- .../neural network/tensorflow/process/nn.py | 39 - .../neural network/tensorflow/process/nn_.py | 39 - .../tensorflow/process/nn_acc.py | 44 -- .../tensorflow/process/nn_attenuate.py | 49 -- .../tensorflow/process/nn_clipping.py | 47 -- .../tensorflow/process/self_LSTM.py | 54 -- .../tensorflow/process/transformer.py | 55 -- .../DL/reduced kernel/kernel_reduced.py | 741 ------------------ .../reduced kernel/process/kernel_reduced.py | 637 --------------- Note 7.0 pv documentation/RL/kernel.txt | 10 - .../RL/neural network/pytorch/DDPG.py | 101 --- .../RL/neural network/pytorch/DQN.py | 64 -- .../RL/neural network/pytorch/DQN_pr.py | 76 -- .../RL/neural network/pytorch/DoubleDQN.py | 65 -- .../RL/neural network/pytorch/DuelingDQN.py | 67 -- .../RL/neural network/tensorflow/DDPG.py | 84 -- .../RL/neural network/tensorflow/DQN.py | 46 -- .../RL/neural network/tensorflow/DQN_pr.py | 59 -- .../neural network/tensorflow/pool net/DQN.py | 52 -- .../tensorflow/pool net/thread/DDPG.py | 84 -- .../tensorflow/pool net/thread/DQN.py | 46 -- .../tensorflow/pool net/thread/DQN_m.py | 48 -- .../tensorflow/pool net/thread/DQN_mtpr.py | 59 -- .../RL/reduced kernel/kernel_reduced.py | 670 ---------------- .../RL/reduced kernel/nspn/kernel_reduced.py | 685 ---------------- Note 7.0 pv documentation/compiler/nc.txt | 33 - Note 7.0 pv documentation/compiler/nn.n | 26 - Note 7.0 pv documentation/compiler/nn.py | 25 - 49 files changed, 5025 deletions(-) delete mode 100644 Note 7.0 pv documentation/DL/neural network/pytorch/nn.py delete mode 100644 Note 7.0 pv documentation/DL/neural network/tensorflow/bert.py delete mode 100644 Note 7.0 pv documentation/DL/neural network/tensorflow/cnn.py delete mode 100644 Note 7.0 pv documentation/DL/neural network/tensorflow/layer/LSTM.py delete mode 100644 Note 7.0 pv documentation/DL/neural network/tensorflow/layer/Transformer.py delete mode 100644 Note 7.0 pv documentation/DL/neural network/tensorflow/layer/attn_LSTM.py delete mode 100644 Note 7.0 pv documentation/DL/neural network/tensorflow/layer/cnn.py delete mode 100644 Note 7.0 pv documentation/DL/neural network/tensorflow/layer/cnn_lmix.py delete mode 100644 Note 7.0 pv documentation/DL/neural network/tensorflow/layer/nn.py delete mode 100644 Note 7.0 pv documentation/DL/neural network/tensorflow/layer/nn_acc.py delete mode 100644 Note 7.0 pv documentation/DL/neural network/tensorflow/layer/nn_clipping.py delete mode 100644 Note 7.0 pv documentation/DL/neural network/tensorflow/layer/self_LSTM.py delete mode 100644 Note 7.0 pv documentation/DL/neural network/tensorflow/lstm.py delete mode 100644 Note 7.0 pv documentation/DL/neural network/tensorflow/nn.py delete mode 100644 Note 7.0 pv documentation/DL/neural network/tensorflow/nn_acc.py delete mode 100644 Note 7.0 pv documentation/DL/neural network/tensorflow/nn_ol.py delete mode 100644 Note 7.0 pv documentation/DL/neural network/tensorflow/process/LSTM.py delete mode 100644 Note 7.0 pv documentation/DL/neural network/tensorflow/process/Transformer.py delete mode 100644 Note 7.0 pv documentation/DL/neural network/tensorflow/process/attn_LSTM.py delete mode 100644 Note 7.0 pv documentation/DL/neural network/tensorflow/process/cnn.py delete mode 100644 Note 7.0 pv documentation/DL/neural network/tensorflow/process/cnn_lmix.py delete mode 100644 Note 7.0 pv documentation/DL/neural network/tensorflow/process/nn.py delete mode 100644 Note 7.0 pv documentation/DL/neural network/tensorflow/process/nn_.py delete mode 100644 Note 7.0 pv documentation/DL/neural network/tensorflow/process/nn_acc.py delete mode 100644 Note 7.0 pv documentation/DL/neural network/tensorflow/process/nn_attenuate.py delete mode 100644 Note 7.0 pv documentation/DL/neural network/tensorflow/process/nn_clipping.py delete mode 100644 Note 7.0 pv documentation/DL/neural network/tensorflow/process/self_LSTM.py delete mode 100644 Note 7.0 pv documentation/DL/neural network/tensorflow/process/transformer.py delete mode 100644 Note 7.0 pv documentation/DL/reduced kernel/kernel_reduced.py delete mode 100644 Note 7.0 pv documentation/DL/reduced kernel/process/kernel_reduced.py delete mode 100644 Note 7.0 pv documentation/RL/kernel.txt delete mode 100644 Note 7.0 pv documentation/RL/neural network/pytorch/DDPG.py delete mode 100644 Note 7.0 pv documentation/RL/neural network/pytorch/DQN.py delete mode 100644 Note 7.0 pv documentation/RL/neural network/pytorch/DQN_pr.py delete mode 100644 Note 7.0 pv documentation/RL/neural network/pytorch/DoubleDQN.py delete mode 100644 Note 7.0 pv documentation/RL/neural network/pytorch/DuelingDQN.py delete mode 100644 Note 7.0 pv documentation/RL/neural network/tensorflow/DDPG.py delete mode 100644 Note 7.0 pv documentation/RL/neural network/tensorflow/DQN.py delete mode 100644 Note 7.0 pv documentation/RL/neural network/tensorflow/DQN_pr.py delete mode 100644 Note 7.0 pv documentation/RL/neural network/tensorflow/pool net/DQN.py delete mode 100644 Note 7.0 pv documentation/RL/neural network/tensorflow/pool net/thread/DDPG.py delete mode 100644 Note 7.0 pv documentation/RL/neural network/tensorflow/pool net/thread/DQN.py delete mode 100644 Note 7.0 pv documentation/RL/neural network/tensorflow/pool net/thread/DQN_m.py delete mode 100644 Note 7.0 pv documentation/RL/neural network/tensorflow/pool net/thread/DQN_mtpr.py delete mode 100644 Note 7.0 pv documentation/RL/reduced kernel/kernel_reduced.py delete mode 100644 Note 7.0 pv documentation/RL/reduced kernel/nspn/kernel_reduced.py delete mode 100644 Note 7.0 pv documentation/compiler/nc.txt delete mode 100644 Note 7.0 pv documentation/compiler/nn.n delete mode 100644 Note 7.0 pv documentation/compiler/nn.py diff --git a/Note 7.0 pv documentation/DL/neural network/pytorch/nn.py b/Note 7.0 pv documentation/DL/neural network/pytorch/nn.py deleted file mode 100644 index d04986b4..00000000 --- a/Note 7.0 pv documentation/DL/neural network/pytorch/nn.py +++ /dev/null @@ -1,53 +0,0 @@ -import torch -from torch import nn - - -class NeuralNetwork(nn.Module): - def __init__(self): - super().__init__() - self.flatten=nn.Flatten() - self.linear_relu_stack=nn.Sequential( - nn.Linear(28*28, 512), - nn.ReLU(), - nn.Linear(512, 512), - nn.ReLU(), - nn.Linear(512, 10) - ) - - - def forward(self,x): - x=self.flatten(x) - logits=self.linear_relu_stack(x) - return logits - - -class neuralnetwork: - def __init__(self): - if torch.cuda.is_available(): - self.device=torch.device('cuda') - else: - self.device=torch.device('cpu') - self.model=NeuralNetwork().to(self.device) - self.loss_fn=nn.CrossEntropyLoss() - self.opt=torch.optim.SGD(self.model.parameters(),lr=1e-3) #optimizer,kernel uses it to optimize. - - - def fp(self,x): #forward propagation function,kernel uses it for forward propagation. - pred=self.model(x.to(self.device)) - return pred - - - def loss(self,output,labels): #loss functino,kernel uses it to calculate loss. - loss=self.loss_fn(output,labels.to(self.device)) - return loss - - - def backward(self,loss): #backward function,kernel uses it for backpropagation. - self.optim.zero_grad() - loss.backward() - return - - - def opt(self): #opt function,kernel uses it to optimize. - self.optim.step() - return diff --git a/Note 7.0 pv documentation/DL/neural network/tensorflow/bert.py b/Note 7.0 pv documentation/DL/neural network/tensorflow/bert.py deleted file mode 100644 index c3dc6588..00000000 --- a/Note 7.0 pv documentation/DL/neural network/tensorflow/bert.py +++ /dev/null @@ -1,167 +0,0 @@ -import tensorflow as tf -import tensorflow_hub as hub - - -class bert: - def __init__(self): - text_input=tf.keras.layers.Input(shape=(),dtype=tf.string,name='text') - preprocessing_layer=hub.KerasLayer(tfhub_handle_preprocess,name='preprocessing') - encoder_inputs=preprocessing_layer(text_input) - encoder=hub.KerasLayer(tfhub_handle_encoder,trainable=True,name='BERT_encoder') - outputs=encoder(encoder_inputs) - net=outputs['pooled_output'] - net=tf.keras.layers.Dropout(0.1)(net) - net=tf.keras.layers.Dense(1,activation=None,name='classifier')(net) - self.model=tf.keras.Model(text_input,net) - self.param=self.model.weights - self.opt=tf.keras.optimizers.Adam() - - - def fp(self,data): - return self.model(data) - - - def loss(self,output,labels): - bce=tf.keras.losses.BinaryCrossentropy(from_logits=True) - return bce(output,labels) - - -bert_model_name = 'small_bert/bert_en_uncased_L-4_H-512_A-8' -map_name_to_handle = { - 'bert_en_uncased_L-12_H-768_A-12': - 'https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/3', - 'bert_en_cased_L-12_H-768_A-12': - 'https://tfhub.dev/tensorflow/bert_en_cased_L-12_H-768_A-12/3', - 'bert_multi_cased_L-12_H-768_A-12': - 'https://tfhub.dev/tensorflow/bert_multi_cased_L-12_H-768_A-12/3', - 'small_bert/bert_en_uncased_L-2_H-128_A-2': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-2_H-128_A-2/1', - 'small_bert/bert_en_uncased_L-2_H-256_A-4': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-2_H-256_A-4/1', - 'small_bert/bert_en_uncased_L-2_H-512_A-8': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-2_H-512_A-8/1', - 'small_bert/bert_en_uncased_L-2_H-768_A-12': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-2_H-768_A-12/1', - 'small_bert/bert_en_uncased_L-4_H-128_A-2': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-128_A-2/1', - 'small_bert/bert_en_uncased_L-4_H-256_A-4': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-256_A-4/1', - 'small_bert/bert_en_uncased_L-4_H-512_A-8': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-512_A-8/1', - 'small_bert/bert_en_uncased_L-4_H-768_A-12': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-768_A-12/1', - 'small_bert/bert_en_uncased_L-6_H-128_A-2': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-6_H-128_A-2/1', - 'small_bert/bert_en_uncased_L-6_H-256_A-4': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-6_H-256_A-4/1', - 'small_bert/bert_en_uncased_L-6_H-512_A-8': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-6_H-512_A-8/1', - 'small_bert/bert_en_uncased_L-6_H-768_A-12': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-6_H-768_A-12/1', - 'small_bert/bert_en_uncased_L-8_H-128_A-2': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-8_H-128_A-2/1', - 'small_bert/bert_en_uncased_L-8_H-256_A-4': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-8_H-256_A-4/1', - 'small_bert/bert_en_uncased_L-8_H-512_A-8': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-8_H-512_A-8/1', - 'small_bert/bert_en_uncased_L-8_H-768_A-12': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-8_H-768_A-12/1', - 'small_bert/bert_en_uncased_L-10_H-128_A-2': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-10_H-128_A-2/1', - 'small_bert/bert_en_uncased_L-10_H-256_A-4': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-10_H-256_A-4/1', - 'small_bert/bert_en_uncased_L-10_H-512_A-8': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-10_H-512_A-8/1', - 'small_bert/bert_en_uncased_L-10_H-768_A-12': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-10_H-768_A-12/1', - 'small_bert/bert_en_uncased_L-12_H-128_A-2': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-12_H-128_A-2/1', - 'small_bert/bert_en_uncased_L-12_H-256_A-4': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-12_H-256_A-4/1', - 'small_bert/bert_en_uncased_L-12_H-512_A-8': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-12_H-512_A-8/1', - 'small_bert/bert_en_uncased_L-12_H-768_A-12': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-12_H-768_A-12/1', - 'albert_en_base': - 'https://tfhub.dev/tensorflow/albert_en_base/2', - 'electra_small': - 'https://tfhub.dev/google/electra_small/2', - 'electra_base': - 'https://tfhub.dev/google/electra_base/2', - 'experts_pubmed': - 'https://tfhub.dev/google/experts/bert/pubmed/2', - 'experts_wiki_books': - 'https://tfhub.dev/google/experts/bert/wiki_books/2', - 'talking-heads_base': - 'https://tfhub.dev/tensorflow/talkheads_ggelu_bert_en_base/1', -} -map_model_to_preprocess = { - 'bert_en_uncased_L-12_H-768_A-12': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'bert_en_cased_L-12_H-768_A-12': - 'https://tfhub.dev/tensorflow/bert_en_cased_preprocess/3', - 'small_bert/bert_en_uncased_L-2_H-128_A-2': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-2_H-256_A-4': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-2_H-512_A-8': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-2_H-768_A-12': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-4_H-128_A-2': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-4_H-256_A-4': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-4_H-512_A-8': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-4_H-768_A-12': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-6_H-128_A-2': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-6_H-256_A-4': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-6_H-512_A-8': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-6_H-768_A-12': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-8_H-128_A-2': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-8_H-256_A-4': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-8_H-512_A-8': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-8_H-768_A-12': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-10_H-128_A-2': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-10_H-256_A-4': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-10_H-512_A-8': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-10_H-768_A-12': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-12_H-128_A-2': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-12_H-256_A-4': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-12_H-512_A-8': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-12_H-768_A-12': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'bert_multi_cased_L-12_H-768_A-12': - 'https://tfhub.dev/tensorflow/bert_multi_cased_preprocess/3', - 'albert_en_base': - 'https://tfhub.dev/tensorflow/albert_en_preprocess/3', - 'electra_small': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'electra_base': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'experts_pubmed': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'experts_wiki_books': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'talking-heads_base': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', -} -tfhub_handle_encoder=map_name_to_handle[bert_model_name] -tfhub_handle_preprocess=map_model_to_preprocess[bert_model_name] diff --git a/Note 7.0 pv documentation/DL/neural network/tensorflow/cnn.py b/Note 7.0 pv documentation/DL/neural network/tensorflow/cnn.py deleted file mode 100644 index 93a2aef1..00000000 --- a/Note 7.0 pv documentation/DL/neural network/tensorflow/cnn.py +++ /dev/null @@ -1,27 +0,0 @@ -import tensorflow as tf -from tensorflow.keras import layers,models - - -class cnn: - def __init__(self): - self.model=models.Sequential() - self.model.add(layers.Conv2D(32,(3,3),activation='relu',input_shape=(32,32,3))) - self.model.add(layers.MaxPooling2D((2,2))) - self.model.add(layers.Conv2D(64,(3,3),activation='relu')) - self.model.add(layers.MaxPooling2D((2,2))) - self.model.add(layers.Conv2D(64,(3,3),activation='relu')) - self.model.add(layers.Flatten()) - self.model.add(layers.Dense(64,activation='relu')) - self.model.add(layers.Dense(10)) - self.param=self.model.weights - self.opt=tf.keras.optimizers.Adam() - - - def fp(self,data): - output=self.model(data) - return output - - - def loss(self,output,labels): - loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - return loss(labels,output) diff --git a/Note 7.0 pv documentation/DL/neural network/tensorflow/layer/LSTM.py b/Note 7.0 pv documentation/DL/neural network/tensorflow/layer/LSTM.py deleted file mode 100644 index 1e52f834..00000000 --- a/Note 7.0 pv documentation/DL/neural network/tensorflow/layer/LSTM.py +++ /dev/null @@ -1,37 +0,0 @@ -import tensorflow as tf -from Note.nn.layer.LSTM import LSTM -from Note.nn.layer.dense import dense - -# Define a recurrent neural network class with LSTM layers -class lstm: - def __init__(self): - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.opt=tf.keras.optimizers.Adam() - - - def build(self): - # Create two LSTM layers with 50 hidden units each - self.lstm1=LSTM(weight_shape=(3,50),return_sequence=True) # First LSTM layer that returns the full sequence - self.lstm2=LSTM(weight_shape=(50,50),) # Second LSTM layer that returns the last output only - # Create a dense layer with 10 output units - self.dense=dense([50, 10]) - # Store the parameters of the layers in a list - self.param=[self.lstm1.param, - self.lstm2.param, - self.dense.param, - ] - return - - - def fp(self,data): - # Perform forward propagation on the input data - x=self.lstm1.output(data) # First LSTM layer output - x=self.lstm2.output(x) # Second LSTM layer output - output=self.dense.output(x) # Dense layer output - return output - - - def loss(self, output, labels): - # Compute the loss between the output and the labels - return self.loss_object(labels, output) diff --git a/Note 7.0 pv documentation/DL/neural network/tensorflow/layer/Transformer.py b/Note 7.0 pv documentation/DL/neural network/tensorflow/layer/Transformer.py deleted file mode 100644 index 5d00ffb7..00000000 --- a/Note 7.0 pv documentation/DL/neural network/tensorflow/layer/Transformer.py +++ /dev/null @@ -1,46 +0,0 @@ -import tensorflow as tf -from Note.nn.layer.Transformer import Transformer -from Note.nn.layer.dense import dense -from Note.nn.layer.emdedding import embedding -from Note.nn.positional_encoding import positional_encoding - -# Define a positional encoding transformer neural network class -class Transformer: - def __init__(self,vocab_size,embed_size,num_heads,num_layers,max_len): - self.vocab_size=vocab_size - self.embed_size=embed_size - self.num_heads=num_heads - self.num_layers=num_layers - self.max_len=max_len - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.opt=tf.keras.optimizers.Adam() - - - def build(self): - # Create an embedding layer to map input tokens to vectors - self.embedding_layer = embedding(self.vocab_size,self.embed_size) - # Create a list of transformer layers - self.transformer_layers = [transformer(weight_shape=[self.embed_size, self.embed_size], num_heads=self.num_heads) for _ in range(self.num_layers)] - # Create a linear layer to map the transformer output to logits - self.logits_layer = dense([self.embed_size,self.vocab_size]) - # Store the parameters of the layers in a list - self.param = [self.embedding_layer.embeddings] - for transformer_layer in self.transformer_layers: - self.param.extend(transformer_layer.param) - self.param.extend(self.logits_layer.param) - - - def fp(self,data): - # Perform forward propagation on the input data - x = self.embedding_layer.output(data) # shape: (batch_size, seq_len, embed_size) - x += positional_encoding(self.max_len,self.embed_size) # shape: (batch_size, seq_len, embed_size) - for transformer_layer in self.transformer_layers: - x = transformer_layer.output(x) # shape: (batch_size, seq_len, embed_size) - output = self.logits_layer.output(x) # shape: (batch_size, seq_len, vocab_size) - return output - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) diff --git a/Note 7.0 pv documentation/DL/neural network/tensorflow/layer/attn_LSTM.py b/Note 7.0 pv documentation/DL/neural network/tensorflow/layer/attn_LSTM.py deleted file mode 100644 index 5d4fa942..00000000 --- a/Note 7.0 pv documentation/DL/neural network/tensorflow/layer/attn_LSTM.py +++ /dev/null @@ -1,44 +0,0 @@ -import tensorflow as tf -from Note.nn.layer.LSTM import LSTM -from Note.nn.layer.dense import dense -from Note.nn.layer.attention import attention - -# Define a recurrent neural network class with LSTM layers and attention -class attn_lstm: - def __init__(self): - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.opt=tf.keras.optimizers.Adam() - - - def build(self): - # Create two LSTM layers with 50 hidden units each - self.lstm1=LSTM(weight_shape=(3,50),return_sequence=True) # First LSTM layer that returns the full sequence - self.lstm2=LSTM(weight_shape=(50,50),) # Second LSTM layer that returns the last output only - # Create a dense layer with 10 output units and softmax activation - self.dense=dense([100, 10],activation='softmax') - # Create an attention object with weight shape (50, 50) - self.attn=attention((50,50)) - # Store the parameters of the layers and the attention in a list - self.param=[self.lstm1.param, - self.lstm2.param, - self.dense.param, - self.attn.param] # Add the attention parameters to the list - return - - - def fp(self,data): - # Perform forward propagation on the input data - x=self.lstm1.output(data) # First LSTM layer output, shape (B, N, 50) - y=self.lstm2.output(x) # Second LSTM layer output, shape (B, 50) - # Use the attention object to compute the context vector from x and y, shape (B, 50) - context_vector,_,_=self.attn.output(x,y) - # Concatenate the context vector and the second LSTM output, shape (B, 100) - z=tf.concat([context_vector,y],axis=-1) - output=self.dense.output(z) # Dense layer output with softmax activation, shape (B, 10) - return output - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) diff --git a/Note 7.0 pv documentation/DL/neural network/tensorflow/layer/cnn.py b/Note 7.0 pv documentation/DL/neural network/tensorflow/layer/cnn.py deleted file mode 100644 index aa2667f8..00000000 --- a/Note 7.0 pv documentation/DL/neural network/tensorflow/layer/cnn.py +++ /dev/null @@ -1,50 +0,0 @@ -import tensorflow as tf -from Note.nn.layer.conv2d import conv2d -from Note.nn.layer.dense import dense -from Note.nn.layer.flatten import flatten - -# Define a convolutional neural network class -class cnn: - def __init__(self): - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.opt=tf.keras.optimizers.Adam() - - - def build(self): - # Create three convolutional layers with relu activations - self.conv1=conv2d(weight_shape=(3,3,3,32),activation='relu') - self.conv2=conv2d(weight_shape=(3,3,32,64),activation='relu') - self.conv3=conv2d(weight_shape=(3,3,64,64),activation='relu') - # Create two dense layers with relu and linear activations - self.dense1=dense([64*4*4,64],activation='relu') - self.dense2=dense([64,10]) - # Store the parameters of the layers in a list - self.param = [self.conv1.weight, - self.conv2.weight, - self.conv3.weight, - self.dense1.weight, - self.dense1.bias, - self.dense2.weight, - self.dense2.bias] - return - - - def fp(self,data): - # Perform forward propagation on the input data - x=tf.nn.conv2d(data,self.conv1.weight,strides=(1,1),padding='SAME') # First convolutional layer - x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # First max pooling layer - x=tf.nn.conv2d(x,self.conv2.weight,strides=(1,1),padding='SAME') # Second convolutional layer - x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # Second max pooling layer - x=tf.nn.conv2d(x,self.conv3.weight,strides=(1,1),padding='SAME') # Third convolutional layer - x=flatten(x) # Flatten the output to a vector - x=tf.nn.relu(tf.matmul(x,self.dense1.weight)+self.dense1.bias) # First dense layer with relu activation - x=tf.nn.dropout(x,rate=0.5) # Apply dropout to prevent overfitting - output=tf.matmul(x,self.dense2.weight)+self.dense2.bias # Output layer with linear activation - output=tf.nn.dropout(output,rate=0.5) # Apply dropout to prevent overfitting - return output - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) \ No newline at end of file diff --git a/Note 7.0 pv documentation/DL/neural network/tensorflow/layer/cnn_lmix.py b/Note 7.0 pv documentation/DL/neural network/tensorflow/layer/cnn_lmix.py deleted file mode 100644 index 2c01b135..00000000 --- a/Note 7.0 pv documentation/DL/neural network/tensorflow/layer/cnn_lmix.py +++ /dev/null @@ -1,58 +0,0 @@ -import tensorflow as tf -from Note.nn.layer.conv2d import conv2d -from Note.nn.layer.dense import dense -from Note.nn.layer.flatten import flatten -from Note.nn.layer.LMix import lmix - -# Define a convolutional neural network class with mixup augmentation -class cnn: - def __init__(self): - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.opt=tf.keras.optimizers.Adam() - # Initialize the alpha parameter for mixup - self.alpha=1.0 - - - def build(self): - # Create three convolutional layers with relu activations - self.conv1=conv2d(weight_shape=(3,3,3,32),activation='relu') - self.conv2=conv2d(weight_shape=(3,3,32,64),activation='relu') - self.conv3=conv2d(weight_shape=(3,3,64,64),activation='relu') - # Create two dense layers with relu and linear activations - self.dense1=dense([64*4*4,64],activation='relu') - self.dense2=dense([64,10]) - # Store the parameters of the layers in a list - self.param = [self.conv1.weight, - self.conv2.weight, - self.conv3.weight, - self.dense1.weight, - self.dense1.bias, - self.dense2.weight, - self.dense2.bias] - return - - - def data_func(self,data_batch,labels_batch): - # Apply mixup augmentation on the input data and labels - return lmix(data_batch,labels_batch,self.alpha) - - - def fp(self,data): - # Perform forward propagation on the input data - x=tf.nn.conv2d(data,self.conv1.weight,strides=(1,1),padding='SAME') # First convolutional layer - x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # First max pooling layer - x=tf.nn.conv2d(x,self.conv2.weight,strides=(1,1),padding='SAME') # Second convolutional layer - x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # Second max pooling layer - x=tf.nn.conv2d(x,self.conv3.weight,strides=(1,1),padding='SAME') # Third convolutional layer - x=flatten(x) # Flatten the output to a vector - x=tf.nn.relu(tf.matmul(x,self.dense1.weight)+self.dense1.bias) # First dense layer with relu activation - x=tf.nn.dropout(x,rate=0.5) # Apply dropout to prevent overfitting - output=tf.matmul(x,self.dense2.weight)+self.dense2.bias # Output layer with linear activation - output=tf.nn.dropout(output,rate=0.5) # Apply dropout to prevent overfitting - return output - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) diff --git a/Note 7.0 pv documentation/DL/neural network/tensorflow/layer/nn.py b/Note 7.0 pv documentation/DL/neural network/tensorflow/layer/nn.py deleted file mode 100644 index 561a9bd5..00000000 --- a/Note 7.0 pv documentation/DL/neural network/tensorflow/layer/nn.py +++ /dev/null @@ -1,27 +0,0 @@ -import tensorflow as tf -import Note.nn.layer.dense as d -from Note.nn.layer.flatten import flatten - - -class nn: - def __init__(self): - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.opt=tf.keras.optimizers.Adam() - - - def build(self): - self.layer1=d.dense([784,128],activation='relu') - self.layer2=d.dense([128,10]) - self.param=[self.layer1.param,self.layer2.param] - return - - - def fp(self,data): - data=flatten(data) - output1=self.layer1.output(data) - output2=self.layer2.output(output1) - return output2 - - - def loss(self,output,labels): - return self.loss_object(labels,output) diff --git a/Note 7.0 pv documentation/DL/neural network/tensorflow/layer/nn_acc.py b/Note 7.0 pv documentation/DL/neural network/tensorflow/layer/nn_acc.py deleted file mode 100644 index cad20961..00000000 --- a/Note 7.0 pv documentation/DL/neural network/tensorflow/layer/nn_acc.py +++ /dev/null @@ -1,32 +0,0 @@ -import tensorflow as tf -import Note.nn.layer.dense as d -from Note.nn.layer.flatten import flatten - - -class nn: - def __init__(self): - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.train_accuracy=tf.keras.metrics.SparseCategoricalAccuracy(name='train_accuracy') - self.opt=tf.keras.optimizers.Adam() - - - def build(self): - self.layer1=d.dense([784,128],activation='relu') - self.layer2=d.dense([128,10]) - self.param=[self.layer1.param,self.layer2.param] - return - - - def fp(self,data): - data=flatten(data) - output1=self.layer1.output(data) - output2=self.layer2.output(output1) - return output2 - - - def loss(self,output,labels): - return self.loss_object(labels,output) - - - def accuracy(self,output,labels): - return self.train_accuracy(labels,output) \ No newline at end of file diff --git a/Note 7.0 pv documentation/DL/neural network/tensorflow/layer/nn_clipping.py b/Note 7.0 pv documentation/DL/neural network/tensorflow/layer/nn_clipping.py deleted file mode 100644 index db0bc108..00000000 --- a/Note 7.0 pv documentation/DL/neural network/tensorflow/layer/nn_clipping.py +++ /dev/null @@ -1,40 +0,0 @@ -import tensorflow as tf -import Note.nn.layer.dense as d -from Note.nn.layer.flatten import flatten -from Note.nn.gradient_clipping import gradient_clipping - -# Define a neural network class -class nn: - def __init__(self): - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.opt=tf.keras.optimizers.Adam() - - - def build(self): - # Create two dense layers with relu and linear activations - self.layer1=d.dense([784,128],activation='relu') - self.layer2=d.dense([128,10]) - # Store the parameters of the layers in a list - self.param=[self.layer1.param,self.layer2.param] - return - - - def fp(self,data): - # Perform forward propagation on the input data - data=flatten(data) - output1=self.layer1.output(data) - output2=self.layer2.output(output1) - return output2 - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) - - - def gradient(self,tape,loss): - # Compute the gradients of the loss with respect to the parameters - grads=tape.gradient(loss,self.param) - # Clip the gradients by value to avoid exploding gradients - return gradient_clipping(grads,'value',1.0) diff --git a/Note 7.0 pv documentation/DL/neural network/tensorflow/layer/self_LSTM.py b/Note 7.0 pv documentation/DL/neural network/tensorflow/layer/self_LSTM.py deleted file mode 100644 index 1b93a9e6..00000000 --- a/Note 7.0 pv documentation/DL/neural network/tensorflow/layer/self_LSTM.py +++ /dev/null @@ -1,45 +0,0 @@ -import tensorflow as tf -from Note.nn.layer.LSTM import LSTM -from Note.nn.layer.self_attention import self_attention -from Note.nn.layer.dense import dense -from Note.nn.layer.emdedding import embedding - -# Define a LSTM self attention neural network class -class self_LSTM: - def __init__(self,vocab_size,embed_size,num_heads,num_layers): - self.vocab_size=vocab_size - self.embed_size=embed_size - self.num_heads=num_heads - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.opt=tf.keras.optimizers.Adam() - - - def build(self): - # Create an embedding layer to map input tokens to vectors - self.embedding_layer = embedding(self.vocab_size,self.embed_size) - # Create a LSTM layer to process the input sequence - self.lstm_layer = LSTM(weight_shape=[self.embed_size,self.embed_size],return_sequence=True) - # Create a self attention layer to attend to the LSTM output - self.self_attention_layer = self_attention(weight_shape=[self.embed_size,self.embed_size],num_heads=self.num_heads) - # Create a linear layer to map the attention output to logits - self.logits_layer = dense([self.embed_size,self.vocab_size]) - # Store the parameters of the layers in a list - self.param = [self.embedding_layer.embeddings] - self.param.extend(self.lstm_layer.param) - self.param.extend(self.self_attention_layer.param) - self.param.extend(self.logits_layer.param) - - - def fp(self,data): - # Perform forward propagation on the input data - x = self.embedding_layer.output(data) # shape: (batch_size, seq_len, embed_size) - x = self.lstm_layer.output(x) # shape: (batch_size, seq_len, embed_size) - x, attention_weights = self.self_attention_layer.output(x,self.num_heads) # shape: (batch_size, seq_len, embed_size) - output = self.logits_layer.output(x) # shape: (batch_size, seq_len, vocab_size) - return output - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) diff --git a/Note 7.0 pv documentation/DL/neural network/tensorflow/lstm.py b/Note 7.0 pv documentation/DL/neural network/tensorflow/lstm.py deleted file mode 100644 index 1fb05875..00000000 --- a/Note 7.0 pv documentation/DL/neural network/tensorflow/lstm.py +++ /dev/null @@ -1,26 +0,0 @@ -import tensorflow as tf - - -class lstm: - def __init__(self,encoder): - self.model=tf.keras.Sequential([ - encoder, - tf.keras.layers.Embedding(len(encoder.get_vocabulary()), 64, mask_zero=True), - tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(64, return_sequences=True)), - tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(32)), - tf.keras.layers.Dense(64, activation='relu'), - tf.keras.layers.Dropout(0.5), - tf.keras.layers.Dense(1) - ]) - self.param=self.model.weights[1:] - self.opt=tf.keras.optimizers.Adam() - - - def fp(self,data): - output=self.model(data) - return output - - - def loss(self,output,labels): - bce=tf.keras.losses.BinaryCrossentropy(from_logits=True) - return bce(labels,output) diff --git a/Note 7.0 pv documentation/DL/neural network/tensorflow/nn.py b/Note 7.0 pv documentation/DL/neural network/tensorflow/nn.py deleted file mode 100644 index 031b75c0..00000000 --- a/Note 7.0 pv documentation/DL/neural network/tensorflow/nn.py +++ /dev/null @@ -1,23 +0,0 @@ -import tensorflow as tf - - -class nn: - def __init__(self): - self.model=tf.keras.models.Sequential([ - tf.keras.layers.Flatten(input_shape=(28, 28)), - tf.keras.layers.Dense(128,activation='relu'), - tf.keras.layers.Dropout(0.2), - tf.keras.layers.Dense(10) - ]) - self.param=self.model.weights #parameter list,kernel uses it list for backpropagation. - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.opt=tf.keras.optimizers.Adam() #optimizer,kernel uses it to optimize. - - - def fp(self,data): #forward propagation function,kernel uses it for forward propagation. - output=self.model(data) - return output - - - def loss(self,output,labels): #loss functino,kernel uses it to calculate loss. - return self.loss_object(labels,output) diff --git a/Note 7.0 pv documentation/DL/neural network/tensorflow/nn_acc.py b/Note 7.0 pv documentation/DL/neural network/tensorflow/nn_acc.py deleted file mode 100644 index 5574e102..00000000 --- a/Note 7.0 pv documentation/DL/neural network/tensorflow/nn_acc.py +++ /dev/null @@ -1,28 +0,0 @@ -import tensorflow as tf - -#An example with accuracy function. -class nn: - def __init__(self): - self.model=tf.keras.models.Sequential([ - tf.keras.layers.Flatten(input_shape=(28, 28)), - tf.keras.layers.Dense(128,activation='relu'), - tf.keras.layers.Dropout(0.2), - tf.keras.layers.Dense(10) - ]) - self.param=self.model.weights #parameter list,kernel uses it list for backpropagation. - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.train_accuracy=tf.keras.metrics.SparseCategoricalAccuracy(name='train_accuracy') - self.opt=tf.keras.optimizers.Adam() #optimizer,kernel uses it to optimize. - - - def fp(self,data): #forward propagation function,kernel uses it for forward propagation. - output=self.model(data) - return output - - - def loss(self,output,labels): #loss functino,kernel uses it to calculate loss. - return self.loss_object(labels,output) - - - def accuracy(self,output,labels): #accuracy function,kernel uses it to calculate accuracy. - return self.train_accuracy(labels,output) diff --git a/Note 7.0 pv documentation/DL/neural network/tensorflow/nn_ol.py b/Note 7.0 pv documentation/DL/neural network/tensorflow/nn_ol.py deleted file mode 100644 index c32bc542..00000000 --- a/Note 7.0 pv documentation/DL/neural network/tensorflow/nn_ol.py +++ /dev/null @@ -1,37 +0,0 @@ -import tensorflow as tf -import numpy as np - -#This is an example of online training. -class nn: - def __init__(self,data,labels): - self.train_data=data - self.train_labels=labels - self.model=tf.keras.models.Sequential([ - tf.keras.layers.Flatten(input_shape=(28, 28)), - tf.keras.layers.Dense(128,activation='relu'), - tf.keras.layers.Dropout(0.2), - tf.keras.layers.Dense(10) - ]) - self.param=self.model.weights #parameter list,kernel uses it list for backpropagation. - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.opt=tf.keras.optimizers.Adam() #optimizer,kernel uses it to optimize. - self.train_loss_list=[] - self.c=0 #counter - self.max_length=1000 - - - def ol(self): #This is simulative online function. - index=np.random.choice(60000,size=[32]) - if self.c==10000: - return 'stop' - else: - return [self.train_data[index],self.train_labels[index]] - - - def fp(self,data): #forward propagation function,kernel uses it for forward propagation. - output=self.model(data) - return output - - - def loss(self,output,labels): #loss functino,kernel uses it to calculate loss. - return self.loss_object(labels,output) \ No newline at end of file diff --git a/Note 7.0 pv documentation/DL/neural network/tensorflow/process/LSTM.py b/Note 7.0 pv documentation/DL/neural network/tensorflow/process/LSTM.py deleted file mode 100644 index 93ac8bf0..00000000 --- a/Note 7.0 pv documentation/DL/neural network/tensorflow/process/LSTM.py +++ /dev/null @@ -1,46 +0,0 @@ -import tensorflow as tf -from Note.nn.layer.LSTM import LSTM -from Note.nn.layer.dense import dense -from Note.nn.process.optimizer import Adam - -# Define a recurrent neural network class with LSTM layers -class lstm: - def __init__(self): - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.optimizer=Adam() - # Initialize a variable to keep track of the batch count - self.bc=tf.Variable(0, dtype=tf.float32) - - - def build(self): - # Create two LSTM layers with 50 hidden units each - self.lstm1=LSTM(weight_shape=(3,50),return_sequence=True) # First LSTM layer that returns the full sequence - self.lstm2=LSTM(weight_shape=(50,50),) # Second LSTM layer that returns the last output only - # Create a dense layer with 10 output units - self.dense=dense([50, 10]) - # Store the parameters of the layers in a list - self.param=[self.lstm1.param, - self.lstm2.param, - self.dense.param, - ] - return - - - def fp(self,data): - # Perform forward propagation on the input data - x=self.lstm1.output(data) # First LSTM layer output - x=self.lstm2.output(x) # Second LSTM layer output - output=self.dense.output(x) # Dense layer output - return output - - - def loss(self, output, labels): - # Compute the loss between the output and the labels - return self.loss_object(labels, output) - - - def opt(self,gradient): - # Perform optimization on the parameters using the gradient and the batch count - param=self.optimizer.opt(gradient,self.param,self.bc[0]) - return param \ No newline at end of file diff --git a/Note 7.0 pv documentation/DL/neural network/tensorflow/process/Transformer.py b/Note 7.0 pv documentation/DL/neural network/tensorflow/process/Transformer.py deleted file mode 100644 index de39b10f..00000000 --- a/Note 7.0 pv documentation/DL/neural network/tensorflow/process/Transformer.py +++ /dev/null @@ -1,55 +0,0 @@ -import tensorflow as tf -import Note.nn.process.optimizer as o -from Note.nn.layer.Transformer import Transformer -from Note.nn.layer.dense import dense -from Note.nn.layer.emdedding import embedding -from Note.nn.positional_encoding import positional_encoding - -# Define a positional encoding transformer neural network class -class Transformer: - def __init__(self,vocab_size,embed_size,num_heads,num_layers,max_len): - self.vocab_size=vocab_size - self.embed_size=embed_size - self.num_heads=num_heads - self.num_layers=num_layers - self.max_len=max_len - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.optimizer=o.Adam() - # Initialize a variable to keep track of the batch count - self.bc=tf.Variable(0,dtype=tf.float32) - - - def build(self): - # Create an embedding layer to map input tokens to vectors - self.embedding_layer = embedding(self.vocab_size,self.embed_size) - # Create a list of transformer layers - self.transformer_layers = [transformer(weight_shape=[self.embed_size, self.embed_size], num_heads=self.num_heads) for _ in range(self.num_layers)] - # Create a linear layer to map the transformer output to logits - self.logits_layer = dense([self.embed_size,self.vocab_size]) - # Store the parameters of the layers in a list - self.param = [self.embedding_layer.embeddings] - for transformer_layer in self.transformer_layers: - self.param.extend(transformer_layer.param) - self.param.extend(self.logits_layer.param) - - - def fp(self,data): - # Perform forward propagation on the input data - x = self.embedding_layer.output(data) # shape: (batch_size, seq_len, embed_size) - x += positional_encoding(self.max_len,self.embed_size) # shape: (batch_size, seq_len, embed_size) - for transformer_layer in self.transformer_layers: - x = transformer_layer.output(x) # shape: (batch_size, seq_len, embed_size) - output = self.logits_layer.output(x) # shape: (batch_size, seq_len, vocab_size) - return output - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) - - - def opt(self,gradient): - # Perform optimization on the parameters using the gradient and the batch count - param=self.optimizer.opt(gradient,self.param,self.bc) - return param diff --git a/Note 7.0 pv documentation/DL/neural network/tensorflow/process/attn_LSTM.py b/Note 7.0 pv documentation/DL/neural network/tensorflow/process/attn_LSTM.py deleted file mode 100644 index 2326ad57..00000000 --- a/Note 7.0 pv documentation/DL/neural network/tensorflow/process/attn_LSTM.py +++ /dev/null @@ -1,53 +0,0 @@ -import tensorflow as tf -from Note.nn.layer.LSTM import LSTM -from Note.nn.layer.dense import dense -from Note.nn.layer.attention import attention -from Note.nn.process.optimizer import Adam - -# Define a recurrent neural network class with LSTM layers and attention -class attn_lstm: - def __init__(self): - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.optimizer=Adam() - # Initialize a variable to keep track of the batch count - self.bc=tf.Variable(0, dtype=tf.float32) - - - def build(self): - # Create two LSTM layers with 50 hidden units each - self.lstm1=LSTM(weight_shape=(3,50),return_sequence=True) # First LSTM layer that returns the full sequence - self.lstm2=LSTM(weight_shape=(50,50),) # Second LSTM layer that returns the last output only - # Create a dense layer with 10 output units and softmax activation - self.dense=dense([100, 10],activation='softmax') - # Create an attention object with weight shape (50, 50) - self.attn=attention((50,50)) - # Store the parameters of the layers and the attention in a list - self.param=[self.lstm1.param, - self.lstm2.param, - self.dense.param, - self.attn.param] # Add the attention parameters to the list - return - - - def fp(self,data): - # Perform forward propagation on the input data - x=self.lstm1.output(data) # First LSTM layer output, shape (B, N, 50) - y=self.lstm2.output(x) # Second LSTM layer output, shape (B, 50) - # Use the attention object to compute the context vector from x and y, shape (B, 50) - context_vector,_,_=self.attn.output(x,y) - # Concatenate the context vector and the second LSTM output, shape (B, 100) - z=tf.concat([context_vector,y],axis=-1) - output=self.dense.output(z) # Dense layer output with softmax activation, shape (B, 10) - return output - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) - - - def opt(self,gradient): - # Perform optimization on the parameters using the gradient and the batch count - param=self.optimizer.opt(gradient,self.param,self.bc[0]) - return param \ No newline at end of file diff --git a/Note 7.0 pv documentation/DL/neural network/tensorflow/process/cnn.py b/Note 7.0 pv documentation/DL/neural network/tensorflow/process/cnn.py deleted file mode 100644 index 49892ccc..00000000 --- a/Note 7.0 pv documentation/DL/neural network/tensorflow/process/cnn.py +++ /dev/null @@ -1,59 +0,0 @@ -import tensorflow as tf -from Note.nn.layer.conv2d import conv2d -from Note.nn.layer.dense import dense -from Note.nn.layer.flatten import flatten -from Note.nn.process.optimizer import Adam - -# Define a convolutional neural network class -class cnn: - def __init__(self): - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.optimizer=Adam() - # Initialize a variable to keep track of the batch count - self.bc=tf.Variable(0,dtype=tf.float32) - - - def build(self): - # Create three convolutional layers with relu activations - self.conv1=conv2d(weight_shape=(3,3,3,32),activation='relu') - self.conv2=conv2d(weight_shape=(3,3,32,64),activation='relu') - self.conv3=conv2d(weight_shape=(3,3,64,64),activation='relu') - # Create two dense layers with relu and linear activations - self.dense1=dense([64*4*4,64],activation='relu') - self.dense2=dense([64,10]) - # Store the parameters of the layers in a list - self.param = [self.conv1.weight, - self.conv2.weight, - self.conv3.weight, - self.dense1.weight, - self.dense1.bias, - self.dense2.weight, - self.dense2.bias] - return - - - def fp(self,data): - # Perform forward propagation on the input data - x=tf.nn.conv2d(data,self.conv1.weight,strides=(1,1),padding='SAME') # First convolutional layer - x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # First max pooling layer - x=tf.nn.conv2d(x,self.conv2.weight,strides=(1,1),padding='SAME') # Second convolutional layer - x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # Second max pooling layer - x=tf.nn.conv2d(x,self.conv3.weight,strides=(1,1),padding='SAME') # Third convolutional layer - x=flatten(x) # Flatten the output to a vector - x=tf.nn.relu(tf.matmul(x,self.dense1.weight)+self.dense1.bias) # First dense layer with relu activation - x=tf.nn.dropout(x,rate=0.5) # Apply dropout to prevent overfitting - output=tf.matmul(x,self.dense2.weight)+self.dense2.bias # Output layer with linear activation - output=tf.nn.dropout(output,rate=0.5) # Apply dropout to prevent overfitting - return output - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) - - - def opt(self,gradient): - # Perform optimization on the parameters using the gradient and the batch count - param=self.optimizer.opt(gradient,self.param,self.bc[0]) - return param \ No newline at end of file diff --git a/Note 7.0 pv documentation/DL/neural network/tensorflow/process/cnn_lmix.py b/Note 7.0 pv documentation/DL/neural network/tensorflow/process/cnn_lmix.py deleted file mode 100644 index c1fa4cc9..00000000 --- a/Note 7.0 pv documentation/DL/neural network/tensorflow/process/cnn_lmix.py +++ /dev/null @@ -1,67 +0,0 @@ -import tensorflow as tf -from Note.nn.layer.conv2d import conv2d -from Note.nn.layer.dense import dense -from Note.nn.layer.flatten import flatten -from Note.nn.process.optimizer import Adam -from Note.nn.layer.LMix import lmix - -# Define a convolutional neural network class with mixup augmentation -class cnn: - def __init__(self): - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.optimizer=Adam() - # Initialize the alpha parameter for mixup - self.alpha=1.0 - # Initialize a variable to keep track of the batch count - self.bc=tf.Variable(0,dtype=tf.float32) - - - def build(self): - # Create three convolutional layers with relu activations - self.conv1=conv2d(weight_shape=(3,3,3,32),activation='relu') - self.conv2=conv2d(weight_shape=(3,3,32,64),activation='relu') - self.conv3=conv2d(weight_shape=(3,3,64,64),activation='relu') - # Create two dense layers with relu and linear activations - self.dense1=dense([64*4*4,64],activation='relu') - self.dense2=dense([64,10]) - # Store the parameters of the layers in a list - self.param = [self.conv1.weight, - self.conv2.weight, - self.conv3.weight, - self.dense1.weight, - self.dense1.bias, - self.dense2.weight, - self.dense2.bias] - return - - - def data_func(self,data_batch,labels_batch): - # Apply mixup augmentation on the input data and labels - return lmix(data_batch,labels_batch,self.alpha) - - - def fp(self,data): - # Perform forward propagation on the input data - x=tf.nn.conv2d(data,self.conv1.weight,strides=(1,1),padding='SAME') # First convolutional layer - x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # First max pooling layer - x=tf.nn.conv2d(x,self.conv2.weight,strides=(1,1),padding='SAME') # Second convolutional layer - x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # Second max pooling layer - x=tf.nn.conv2d(x,self.conv3.weight,strides=(1,1),padding='SAME') # Third convolutional layer - x=flatten(x) # Flatten the output to a vector - x=tf.nn.relu(tf.matmul(x,self.dense1.weight)+self.dense1.bias) # First dense layer with relu activation - x=tf.nn.dropout(x,rate=0.5) # Apply dropout to prevent overfitting - output=tf.matmul(x,self.dense2.weight)+self.dense2.bias # Output layer with linear activation - output=tf.nn.dropout(output,rate=0.5) # Apply dropout to prevent overfitting - return output - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) - - - def opt(self,gradient): - # Perform optimization on the parameters using the gradient and the batch count - param=self.optimizer.opt(gradient,self.param,self.bc[0]) - return param \ No newline at end of file diff --git a/Note 7.0 pv documentation/DL/neural network/tensorflow/process/nn.py b/Note 7.0 pv documentation/DL/neural network/tensorflow/process/nn.py deleted file mode 100644 index 70f2a637..00000000 --- a/Note 7.0 pv documentation/DL/neural network/tensorflow/process/nn.py +++ /dev/null @@ -1,39 +0,0 @@ -import tensorflow as tf -import Note.nn.layer.dense as d -from Note.nn.process.optimizer import Momentum -from Note.nn.layer.flatten import flatten - -# Define a neural network class -class nn: - def __init__(self): - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.optimizer=Momentum(0.07,0.7) - - - def build(self): - # Create two dense layers with relu and linear activations - self.layer1=d.dense([784,128],activation='relu') - self.layer2=d.dense([128,10]) - # Store the parameters of the layers in a list - self.param=[self.layer1.param,self.layer2.param] - return - - - def fp(self,data): - # Perform forward propagation on the input data - data=flatten(data) - output1=self.layer1.output(data) - output2=self.layer2.output(output1) - return output2 - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) - - - def opt(self,gradient): - # Perform optimization on the parameters using the gradient - param=self.optimizer.opt(gradient,self.param) - return param \ No newline at end of file diff --git a/Note 7.0 pv documentation/DL/neural network/tensorflow/process/nn_.py b/Note 7.0 pv documentation/DL/neural network/tensorflow/process/nn_.py deleted file mode 100644 index 3f50c1e3..00000000 --- a/Note 7.0 pv documentation/DL/neural network/tensorflow/process/nn_.py +++ /dev/null @@ -1,39 +0,0 @@ -import tensorflow as tf -from Note.nn.process.optimizer import Momentum -from Note.nn.layer.flatten import flatten - -# Define a neural network class -class nn: - def __init__(self): - # Initialize the weights and biases of three layers with random values - self.weight1=tf.Variable(tf.random.normal([784,64],dtype='float64')) - self.bias1=tf.Variable(tf.random.normal([64],dtype='float64')) - self.weight2=tf.Variable(tf.random.normal([64,64],dtype='float64')) - self.bias2=tf.Variable(tf.random.normal([64],dtype='float64')) - self.weight3=tf.Variable(tf.random.normal([64,10],dtype='float64')) - self.bias3=tf.Variable(tf.random.normal([10],dtype='float64')) - # Store the parameters of the layers in a list - self.param=[self.weight1,self.weight2,self.weight3,self.bias1,self.bias2,self.bias3] - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.optimizer=Momentum(0.07,0.7) - - - def fp(self,data): - # Perform forward propagation on the input data - data=flatten(data) - layer1=tf.nn.relu(tf.matmul(data,self.weight1)+self.bias1) # First layer with relu activation - layer2=tf.nn.relu(tf.matmul(layer1,self.weight2)+self.bias2) # Second layer with relu activation - output=tf.matmul(layer2,self.weight3)+self.bias3 # Output layer with linear activation - return output - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) - - - def opt(self,gradient): - # Perform optimization on the parameters using the gradient - param=self.optimizer.opt(gradient,self.param) - return param \ No newline at end of file diff --git a/Note 7.0 pv documentation/DL/neural network/tensorflow/process/nn_acc.py b/Note 7.0 pv documentation/DL/neural network/tensorflow/process/nn_acc.py deleted file mode 100644 index 0647360e..00000000 --- a/Note 7.0 pv documentation/DL/neural network/tensorflow/process/nn_acc.py +++ /dev/null @@ -1,44 +0,0 @@ -import tensorflow as tf -import Note.nn.layer.dense as d -from Note.nn.process.optimizer import Momentum -from Note.nn.layer.flatten import flatten -from Note.nn.accuracy import sparse_categorical_accuracy - -# Define a neural network class -class nn: - def __init__(self): - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.optimizer=Momentum(0.07,0.7) - - - def build(self): - # Create two dense layers with relu and linear activations - self.layer1=d.dense([784,128],activation='relu') - self.layer2=d.dense([128,10]) - # Store the parameters of the layers in a list - self.param=[self.layer1.param,self.layer2.param] - return - - - def fp(self,data): - # Perform forward propagation on the input data - data=flatten(data) - output1=self.layer1.output(data) - output2=self.layer2.output(output1) - return output2 - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) - - - def accuracy(self,output,labels): #accuracy function,kernel uses it to calculate accuracy. - return sparse_categorical_accuracy(labels,output) - - - def opt(self,gradient): - # Perform optimization on the parameters using the gradient - param=self.optimizer.opt(gradient,self.param) - return param diff --git a/Note 7.0 pv documentation/DL/neural network/tensorflow/process/nn_attenuate.py b/Note 7.0 pv documentation/DL/neural network/tensorflow/process/nn_attenuate.py deleted file mode 100644 index 890b3335..00000000 --- a/Note 7.0 pv documentation/DL/neural network/tensorflow/process/nn_attenuate.py +++ /dev/null @@ -1,49 +0,0 @@ -import tensorflow as tf -import Note.nn.layer.dense as d -from Note.nn.process.optimizer import Momentum -from Note.nn.layer.flatten import flatten - -# Define a neural network class with gradient attenuation -class nn: - def __init__(self): - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.optimizer=Momentum(0.07,0.7) - # Initialize a variable to keep track of the number of optimization steps - self.opt_counter=tf.Variable(tf.zeros(7,dtype=tf.float32)) - - - def build(self): - # Create two dense layers with relu and linear activations - self.layer1=d.dense([784,128],activation='relu') - self.layer2=d.dense([128,10]) - # Store the parameters of the layers in a list - self.param=[self.layer1.param,self.layer2.param] - return - - - def fp(self,data): - # Perform forward propagation on the input data - data=flatten(data) - output1=self.layer1.output(data) - output2=self.layer2.output(output1) - return output2 - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) - - - def attenuate(self,gradient,oc,p): #gradient attenuation function,kernel uses it to calculate attenuation coefficient. - # Apply an exponential decay to the gradient based on the optimization counter - ac=0.9**oc[p] #ac:attenuation coefficient - for i in range(len(gradient)): #oc:optimization counter - gradient[i]=ac*gradient[i] #p:process number - return gradient - - - def opt(self,gradient): - # Perform optimization on the parameters using the gradient - param=self.optimizer.opt(gradient,self.param) - return param \ No newline at end of file diff --git a/Note 7.0 pv documentation/DL/neural network/tensorflow/process/nn_clipping.py b/Note 7.0 pv documentation/DL/neural network/tensorflow/process/nn_clipping.py deleted file mode 100644 index 32779f84..00000000 --- a/Note 7.0 pv documentation/DL/neural network/tensorflow/process/nn_clipping.py +++ /dev/null @@ -1,47 +0,0 @@ -import tensorflow as tf -import Note.nn.layer.dense as d -from Note.nn.process.optimizer import Momentum -from Note.nn.layer.flatten import flatten -from Note.nn.gradient_clipping import gradient_clipping - -# Define a neural network class -class nn: - def __init__(self): - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.optimizer=Momentum(0.07,0.7) - - - def build(self): - # Create two dense layers with relu and linear activations - self.layer1=d.dense([784,128],activation='relu') - self.layer2=d.dense([128,10]) - # Store the parameters of the layers in a list - self.param=[self.layer1.param,self.layer2.param] - return - - - def fp(self,data): - # Perform forward propagation on the input data - data=flatten(data) - output1=self.layer1.output(data) - output2=self.layer2.output(output1) - return output2 - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) - - - def gradient(self,tape,loss): - # Compute the gradients of the loss with respect to the parameters - grads=tape.gradient(loss,self.param) - # Clip the gradients by value to avoid exploding gradients - return gradient_clipping(grads,'value',1.0) - - - def opt(self,gradient): - # Perform optimization on the parameters using the gradient - param=self.optimizer.opt(gradient,self.param) - return param \ No newline at end of file diff --git a/Note 7.0 pv documentation/DL/neural network/tensorflow/process/self_LSTM.py b/Note 7.0 pv documentation/DL/neural network/tensorflow/process/self_LSTM.py deleted file mode 100644 index a25f4043..00000000 --- a/Note 7.0 pv documentation/DL/neural network/tensorflow/process/self_LSTM.py +++ /dev/null @@ -1,54 +0,0 @@ -import tensorflow as tf -from Note.nn.process.optimizer import Adam -from Note.nn.layer.LSTM import LSTM -from Note.nn.layer.self_attention import self_attention -from Note.nn.layer.dense import dense -from Note.nn.layer.emdedding import embedding - -# Define a LSTM self attention neural network class -class self_LSTM: - def __init__(self,vocab_size,embed_size,num_heads,num_layers): - self.vocab_size=vocab_size - self.embed_size=embed_size - self.num_heads=num_heads - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.optimizer=Adam() - # Initialize a variable to keep track of the batch count - self.bc=tf.Variable(0,dtype=tf.float32) - - - def build(self): - # Create an embedding layer to map input tokens to vectors - self.embedding_layer = embedding(self.vocab_size,self.embed_size) - # Create a LSTM layer to process the input sequence - self.lstm_layer = LSTM(weight_shape=[self.embed_size,self.embed_size],return_sequence=True) - # Create a self attention layer to attend to the LSTM output - self.self_attention_layer = self_attention(weight_shape=[self.embed_size,self.embed_size],num_heads=self.num_heads) - # Create a linear layer to map the attention output to logits - self.logits_layer = dense([self.embed_size,self.vocab_size]) - # Store the parameters of the layers in a list - self.param = [self.embedding_layer.embeddings] - self.param.extend(self.lstm_layer.param) - self.param.extend(self.self_attention_layer.param) - self.param.extend(self.logits_layer.param) - - - def fp(self,data): - # Perform forward propagation on the input data - x = self.embedding_layer.output(data) # shape: (batch_size, seq_len, embed_size) - x = self.lstm_layer.output(x) # shape: (batch_size, seq_len, embed_size) - x, attention_weights = self.self_attention_layer.output(x,self.num_heads) # shape: (batch_size, seq_len, embed_size) - output = self.logits_layer.output(x) # shape: (batch_size, seq_len, vocab_size) - return output - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) - - - def opt(self,gradient): - # Perform optimization on the parameters using the gradient and the batch count - param=self.optimizer.opt(gradient,self.param,self.bc[0]) - return param \ No newline at end of file diff --git a/Note 7.0 pv documentation/DL/neural network/tensorflow/process/transformer.py b/Note 7.0 pv documentation/DL/neural network/tensorflow/process/transformer.py deleted file mode 100644 index a77978a6..00000000 --- a/Note 7.0 pv documentation/DL/neural network/tensorflow/process/transformer.py +++ /dev/null @@ -1,55 +0,0 @@ -import tensorflow as tf -from Note.nn.process.optimizer import Adam -from Note.nn.layer.transformer import transformer -from Note.nn.layer.dense import dense -from Note.nn.layer.emdedding import embedding -from Note.nn.positional_encoding import positional_encoding - -# Define a positional encoding transformer neural network class -class Transformer: - def __init__(self,vocab_size,embed_size,num_heads,num_layers,max_len): - self.vocab_size=vocab_size - self.embed_size=embed_size - self.num_heads=num_heads - self.num_layers=num_layers - self.max_len=max_len - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.optimizer=Adam() - # Initialize a variable to keep track of the batch count - self.bc=tf.Variable(0,dtype=tf.float32) - - - def build(self): - # Create an embedding layer to map input tokens to vectors - self.embedding_layer = embedding(self.vocab_size,self.embed_size) - # Create a list of transformer layers - self.transformer_layers = [transformer(weight_shape=[self.embed_size, self.embed_size], num_heads=self.num_heads) for _ in range(self.num_layers)] - # Create a linear layer to map the transformer output to logits - self.logits_layer = dense([self.embed_size,self.vocab_size]) - # Store the parameters of the layers in a list - self.param = [self.embedding_layer.embeddings] - for transformer_layer in self.transformer_layers: - self.param.extend(transformer_layer.param) - self.param.extend(self.logits_layer.param) - - - def fp(self,data): - # Perform forward propagation on the input data - x = self.embedding_layer.output(data) # shape: (batch_size, seq_len, embed_size) - x += positional_encoding(self.max_len,self.embed_size) # shape: (batch_size, seq_len, embed_size) - for transformer_layer in self.transformer_layers: - x = transformer_layer.output(x) # shape: (batch_size, seq_len, embed_size) - output = self.logits_layer.output(x) # shape: (batch_size, seq_len, vocab_size) - return output - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) - - - def opt(self,gradient): - # Perform optimization on the parameters using the gradient and the batch count - param=self.optimizer.opt(gradient,self.param,self.bc[0]) - return param \ No newline at end of file diff --git a/Note 7.0 pv documentation/DL/reduced kernel/kernel_reduced.py b/Note 7.0 pv documentation/DL/reduced kernel/kernel_reduced.py deleted file mode 100644 index 856f99cd..00000000 --- a/Note 7.0 pv documentation/DL/reduced kernel/kernel_reduced.py +++ /dev/null @@ -1,741 +0,0 @@ -from tensorflow import function -import numpy as np -import matplotlib.pyplot as plt -import time - - -class kernel: - def __init__(self,nn=None): - self.nn=nn # the neural network object - try: - self.nn.km=1 # a flag to indicate the kernel mode - except Exception: - pass - self.platform=None # the platform to use, either tensorflow or pytorch - self.batches=None # the number of batches for training data - self.suspend=False # a flag to indicate whether to suspend the training - self.stop=False # a flag to indicate whether to stop the training - self.stop_flag=False # a flag to indicate whether the training has been stopped - self.save_epoch=None # the epoch number to save the model - self.batch=None # the batch size for training data - self.epoch=0 # the current epoch number - self.end_loss=None # the target loss value to end the training - self.end_acc=None # the target accuracy value to end the training - self.end_test_loss=None # the target test loss value to end the training - self.end_test_acc=None # the target test accuracy value to end the training - self.acc_flag='%' # a flag to indicate whether to use percentage or decimal for accuracy display - self.train_counter=0 # a counter for how many times the train method has been called - self.filename='save.dat' # the file name to save the model - self.train_loss=None # the current train loss value - self.train_acc=None # the current train accuracy value - self.train_loss_list=[] # a list of train loss values for each epoch - self.train_acc_list=[] # a list of train accuracy values for each epoch - self.test_loss=None # the current test loss value - self.test_acc=None # the current test accuracy value - self.test_loss_list=[] # a list of test loss values for each epoch - self.test_acc_list=[] # a list of test accuracy values for each epoch - self.test_flag=False # a flag to indicate whether to use test data or not - self.total_epoch=0 # the total number of epochs for all trainings - self.time=0 # the time elapsed for one training session - self.total_time=0 # the total time elapsed for all trainings - - - def data(self,train_data=None,train_labels=None,test_data=None,test_labels=None,train_dataset=None,test_dataset=None): - if type(self.nn.param[0])!=list: - self.train_data=train_data.astype(self.nn.param[0].dtype.name) # convert the train data type to match the neural network parameters type - self.train_labels=train_labels.astype(self.nn.param[0].dtype.name) # convert the train labels type to match the neural network parameters type - else: - self.train_data=train_data.astype(self.nn.param[0][0].dtype.name) # convert the train data type to match the neural network parameters type (for multiple inputs) - self.train_labels=train_labels.astype(self.nn.param[0][0].dtype.name) # convert the train labels type to match the neural network parameters type (for multiple inputs) - self.train_dataset=train_dataset # a tensorflow or pytorch dataset object for train data (optional) - if test_data is not None: - if type(self.nn.param[0])!=list: - self.test_data=test_data.astype(self.nn.param[0].dtype.name) # convert the test data type to match the neural network parameters type - self.test_labels=test_labels.astype(self.nn.param[0].dtype.name) # convert the test labels type to match the neural network parameters type - else: - self.test_data=test_data.astype(self.nn.param[0][0].dtype.name) # convert the test data type to match the neural network parameters type (for multiple inputs) - self.test_labels=test_labels.astype(self.nn.param[0][0].dtype.name) # convert the test labels type to match the neural network parameters type (for multiple inputs) - self.test_flag=True # set the test flag to True if test data is provided - self.test_dataset=test_dataset # a tensorflow or pytorch dataset object for test data (optional) - if self.train_dataset==None: - self.shape0=train_data.shape[0] # get the number of samples in train data - return - - - def init(self): # a method to initialize the attributes for a new training session - self.suspend=False - self.stop=False - self.stop_flag=False - self.save_epoch=None - self.end_loss=None - self.end_acc=None - self.end_test_loss=None - self.end_test_acc=None - self.train_loss=None - self.train_acc=None - self.test_loss=None - self.test_acc=None - self.train_loss_list.clear() - self.train_acc_list.clear() - self.test_loss_list.clear() - self.test_acc_list.clear() - self.test_flag=False - self.train_counter=0 - self.epoch=0 - self.total_epoch=0 - self.time=0 - self.total_time=0 - return - - - def end(self): # a method to check whether the training has reached the target loss or accuracy values - if self.end_loss!=None and len(self.train_loss_list)!=0 and self.train_loss_list[-1]self.end_acc: # if the target train accuracy is given and the current train accuracy is higher than it - return True - elif self.end_loss!=None and len(self.train_loss_list)!=0 and self.end_acc!=None and self.train_loss_list[-1]self.end_acc: # if both the target train loss and accuracy are given and the current train loss is lower than it and the current train accuracy is higher than it - return True - elif self.end_test_loss!=None and len(self.test_loss_list)!=0 and self.test_loss_list[-1]self.end_test_acc: # if the target test accuracy is given and the current test accuracy is higher than it - return True - elif self.end_test_loss!=None and len(self.test_loss_list)!=0 and self.end_test_acc!=None and self.test_loss_list[-1]self.end_test_acc: # if both the target test loss and accuracy are given and the current test loss is lower than it and the current test accuracy is higher than it - return True - - - def loss_acc(self,output=None,labels_batch=None,loss=None,test_batch=None,total_loss=None,total_acc=None): # a method to calculate the loss and accuracy values for each batch or epoch - if self.batch!=None: # if batch mode is used - total_loss+=loss # accumulate the batch loss to total loss - try: - batch_acc=self.nn.accuracy(output,labels_batch) # calculate the batch accuracy using the neural network's accuracy method - total_acc+=batch_acc # accumulate the batch accuracy to total accuracy - except Exception as e: - try: - if self.nn.accuracy!=None: # if the neural network has an accuracy method defined - raise e # raise the exception - except Exception: - pass - return total_loss,total_acc # return the total loss and accuracy values for all batches so far - else: # if batch mode is not used (use all data at once) - loss=loss.numpy() # convert the loss value to numpy array - self.train_loss=loss # assign the loss value to train loss attribute - self.train_loss_list.append(loss) # append the loss value to train loss list - try: - acc=self.nn.accuracy(output,self.train_labels) # calculate the accuracy value using the neural network's accuracy method - acc=acc.numpy() # convert the accuracy value to numpy array - self.train_acc=acc # assign the accuracy value to train accuracy attribute - self.train_acc_list.append(acc) # append the accuracy value to train accuracy list - except Exception as e: - try: - if self.nn.accuracy!=None: # if the neural network has an accuracy method defined - raise e # raise the exception - except Exception: - pass - if self.test_flag==True: # if test data is used - self.test_loss,self.test_acc=self.test(self.test_data,self.test_labels,test_batch) # calculate the test loss and accuracy values using the test method - self.test_loss_list.append(self.test_loss) # append the test loss value to test loss list - try: - self.test_acc_list.append(self.test_acc) # append the test accuracy value to test accuracy list - except Exception as e: - try: - if self.nn.accuracy!=None: # if the neural network has an accuracy method defined - raise e # raise the exception - except Exception: - pass - return # return nothing for this case - - - def data_func(self,data_batch=None,labels_batch=None,batch=None,index1=None,index2=None,j=None,flag=None): # a method to get a batch of data and labels from the train data and labels - if flag==None: # if flag is None, it means the batch size is smaller than the number of samples - if batch!=1: # if batch size is not 1 - data_batch=self.train_data[index1:index2] # get a slice of train data according to the index range - else: # if batch size is 1 - data_batch=self.train_data[j] # get one sample of train data according to the index - if batch!=1: # if batch size is not 1 - labels_batch=self.train_labels[index1:index2] # get a slice of train labels according to the index range - else: # if batch size is 1 - labels_batch=self.train_labels[j] # get one sample of train labels according to the index - else: # if flag is not None, it means the batch size is larger than the number of samples - try: - try: - data_batch=self.platform.concat([self.train_data[index1:],self.train_data[:index2]],0) # concatenate two slices of train data from the end and the beginning to form a batch - labels_batch=self.platform.concat([self.train_labels[index1:],self.train_labels[:index2]],0) # concatenate two slices of train labels from the end and the beginning to form a batch - except Exception: # if the platform's concat method fails - data_batch=np.concatenate([self.train_data[index1:],self.train_data[:index2]],0) # use numpy's concatenate method instead - labels_batch=np.concatenate([self.train_labels[index1:],self.train_labels[:index2]],0) # use numpy's concatenate method instead - except Exception as e: - raise e # raise any other exception - return data_batch,labels_batch # return the batch of data and labels - - - @function(jit_compile=True) # use tensorflow's function decorator to speed up the execution - def tf_opt(self,data,labels): # a method to perform one optimization step using tensorflow platform - try: - try: - if self.nn.GradientTape!=None: # if the neural network has a GradientTape method defined - tape,output,loss=self.nn.GradientTape(data,labels) # use the neural network's GradientTape method to get the tape, output and loss values - except Exception: # if the neural network does not have a GradientTape method defined or it fails - with self.platform.GradientTape(persistent=True) as tape: # use tensorflow's GradientTape context manager instead - try: - output=self.nn.fp(data) # get the output value using the neural network's forward propagation method - loss=self.nn.loss(output,labels) # get the loss value using the neural network's loss function - except Exception: # if the neural network's forward propagation method or loss function fails or they are combined in one method - output,loss=self.nn.fp(data,labels) # use the neural network's forward propagation method with both data and labels as inputs to get the output and loss values - except Exception as e: - raise e # raise any other exception - try: - try: - gradient=self.nn.gradient(tape,loss) # use the neural network's gradient method to get the gradient value from the tape and loss values - except Exception: # if the neural network does not have a gradient method defined or it fails - gradient=tape.gradient(loss,self.nn.param) # use tensorflow's tape.gradient method instead with loss value and neural network parameters as inputs - except Exception as e: - raise e # raise any other exception - try: - try: - self.nn.opt.apply_gradients(zip(gradient,self.nn.param)) # use the neural network's optimizer's apply_gradients method to update the neural network parameters with gradient value - except Exception: # if the neural network does not have an optimizer or its apply_gradients method fails - self.nn.opt(gradient) # use the neural network's optimizer directly with gradient value as input - except Exception as e: - raise e # raise any other exception - return output,loss # return the output and loss values for this optimization step - - - def pytorch_opt(self,data,labels): # a method to perform one optimization step using pytorch platform - output=self.nn.fp(data) # get the output value using the neural network's forward propagation method - loss=self.nn.loss(output,labels) # get the loss value using the neural network's loss function - try: - try: - self.nn.opt.zero_grad() # use the neural network's optimizer's zero_grad method to clear the previous gradients - loss.backward() # use pytorch's loss.backward method to calculate the gradients - self.nn.opt.step() # use the neural network's optimizer's step method to update the neural network parameters with gradient value - except Exception: # if the neural network does not have an optimizer or its zero_grad or step method fails - self.nn.opt(loss) # use the neural network's optimizer directly with loss value as input - except Exception as e: - raise e # raise any other exception - return output,loss # return the output and loss values for this optimization step - - - def opt(self,data,labels): # a method to perform one optimization step using either tensorflow or pytorch platform - try: - try: - if self.platform.DType!=None: # if tensorflow platform is used - output,loss=self.tf_opt(data,labels) # use the tf_opt method for optimization - except Exception: # if tensorflow platform is not used or it fails - output,loss=self.pytorch_opt(data,labels) # use the pytorch_opt method for optimization - except Exception as e: - raise e # raise any other exception - return output,loss # return the output and loss values for this optimization step - - - def _train(self,batch=None,test_batch=None): # a method to perform one epoch of training - if batch!=None: # if batch mode is used - total_loss=0 # initialize the total loss value for all batches - total_acc=0 # initialize the total accuracy value for all batches - if self.train_dataset!=None: # if a tensorflow or pytorch dataset object is used for train data - for data_batch,labels_batch in self.train_dataset: # iterate over each batch of data and labels from the train dataset - if self.stop==True: # if the stop flag is set to True - if self.stop_func(): # check whether the training has reached the target loss or accuracy values using the stop_func method - return # return nothing and end the training - try: - data_batch,labels_batch=self.nn.data_func(data_batch,labels_batch) # use the neural network's data_func method to preprocess the data and labels batch (optional) - except Exception as e: - try: - if self.nn.data_func!=None: # if the neural network has a data_func method defined - raise e # raise the exception - except Exception: - pass - output,batch_loss=self.opt(data_batch,labels_batch) # perform one optimization step using the opt method and get the output and batch loss values - total_loss,total_acc=self.loss_acc(output=output,labels_batch=labels_batch,loss=batch_loss,total_loss=total_loss,total_acc=total_acc) # calculate and update the total loss and accuracy values for all batches using the loss_acc method - else: # if a numpy array is used for train data - total_loss=0 # initialize the total loss value for all batches - total_acc=0 # initialize the total accuracy value for all batches - batches=int((self.shape0-self.shape0%batch)/batch) # calculate how many batches are needed for train data according to batch size - for j in range(batches): # iterate over each batch index - if self.stop==True: # if the stop flag is set to True - if self.stop_func(): # check whether the training has reached the target loss or accuracy values using the stop_func method - return # return nothing and end the training - index1=j*batch # calculate the start index of train data for this batch - index2=(j+1)*batch # calculate the end index of train data for this batch - data_batch,labels_batch=self.data_func(batch,index1,index2,j) # get a batch of data and labels from train data and labels using the data_func method - try: - data_batch,labels_batch=self.nn.data_func(data_batch,labels_batch) # use the neural network's data_func method to preprocess the data and labels batch (optional) - except Exception as e: - try: - if self.nn.data_func!=None: # if the neural network has a data_func method defined - raise e # raise the exception - except Exception: - pass - output,batch_loss=self.opt(data_batch,labels_batch) # perform one optimization step - total_loss,total_acc=self.loss_acc(output=output,labels_batch=labels_batch,loss=batch_loss,total_loss=total_loss,total_acc=total_acc) # calculate and update the total loss and accuracy values for all batches using the loss_acc method - try: - try: - self.nn.bc.assign_add(1) # use the neural network's batch counter's assign_add method to increase the batch counter by 1 (for tensorflow platform) - except Exception: # if the neural network does not have a batch counter or its assign_add method fails - self.nn.bc+=1 # use the normal addition operation to increase the batch counter by 1 (for pytorch platform) - except Exception: - pass - if self.shape0%batch!=0: # if there are some samples left in train data that are not enough to form a full batch - if self.stop==True: # if the stop flag is set to True - if self.stop_func(): # check whether the training has reached the target loss or accuracy values using the stop_func method - return # return nothing and end the training - batches+=1 # increase the number of batches by 1 to include the remaining samples - index1=batches*batch # calculate the start index of train data for the last batch - index2=batch-(self.shape0-batches*batch) # calculate how many samples are needed from the beginning of train data to form a full batch - data_batch,labels_batch=self.data_func(batch,index1,index2,flag=True) # get a batch of data and labels from train data and labels using the data_func method with flag set to True - try: - data_batch,labels_batch=self.nn.data_func(data_batch,labels_batch) # use the neural network's data_func method to preprocess the data and labels batch (optional) - except Exception as e: - try: - if self.nn.data_func!=None: # if the neural network has a data_func method defined - raise e # raise the exception - except Exception: - pass - output,batch_loss=self.opt(data_batch,labels_batch) # perform one optimization step using the opt method and get the output and batch loss values - total_loss,total_acc=self.loss_acc(output=output,labels_batch=labels_batch,loss=batch_loss,total_loss=total_loss,total_acc=total_acc) # calculate and update the total loss and accuracy values for all batches using the loss_acc method - try: - try: - self.nn.bc.assign_add(1) # use the neural network's batch counter's assign_add method to increase the batch counter by 1 (for tensorflow platform) - except Exception: # if the neural network does not have a batch counter or its assign_add method fails - self.nn.bc+=1 # use the normal addition operation to increase the batch counter by 1 (for pytorch platform) - except Exception: - pass - try: - if self.platform.DType!=None: # if tensorflow platform is used - loss=total_loss.numpy()/batches # convert the total loss value to numpy array and divide it by number of batches to get the average loss value for this epoch - except Exception: # if tensorflow platform is not used or it fails - loss=total_loss.detach().numpy()/batches # detach the total loss value from computation graph and convert it to numpy array and divide it by number of batches to get the average loss value for this epoch - try: - train_acc=total_acc.numpy()/batches # convert the total accuracy value to numpy array and divide it by number of batches to get the average accuracy value for this epoch - except Exception as e: - try: - if self.nn.accuracy!=None: # if the neural network has an accuracy method defined - raise e # raise the exception - except Exception: - pass - self.train_loss=loss # assign the average loss value to train loss attribute - self.train_loss_list.append(loss) # append the average loss value to train loss list - try: - self.train_acc=train_acc # assign the average accuracy value to train accuracy attribute - self.train_acc_list.append(train_acc) # append the average accuracy value to train accuracy list - except Exception as e: - try: - if self.nn.accuracy!=None: # if the neural network has an accuracy method defined - raise e # raise the exception - except Exception: - pass - if self.test_flag==True: # if test data is used - self.test_loss,self.test_acc=self.test(self.test_data,self.test_labels,test_batch) # calculate the test loss and accuracy values using the test method - self.test_loss_list.append(self.test_loss) # append the test loss value to test loss list - try: - self.test_acc_list.append(self.test_acc) # append the test accuracy value to test accuracy list - except Exception as e: - try: - if self.nn.accuracy!=None: # if the neural network has an accuracy method defined - raise e # raise the exception - except Exception: - pass - else: # if batch mode is not used (use all data at once) - output,train_loss=self.opt(self.train_data,self.train_labels) # perform one optimization step using the opt method and get the output and train loss values - self.loss_acc(output=output,labels_batch=labels_batch,loss=train_loss,test_batch=test_batch,total_loss=total_loss,total_acc=total_acc) # calculate and update the train and test loss and accuracy values using the loss_acc method - return # return nothing for this case - - - def train(self,batch=None,epoch=None,test_batch=None,save=None,one=True,p=None,s=None): # a method to perform multiple epochs of training - self.batch=batch # assign the batch size for training data to batch attribute - self.epoch=0 # initialize the current epoch number to 0 - self.train_counter+=1 # increase the train counter by 1 - if p==None: # if p is None, it means the default value of p is used - self.p=9 # assign 9 to p attribute, which means print the train and test loss and accuracy values every 10 epochs - else: - self.p=p-1 # assign p-1 to p attribute, which means print the train and test loss and accuracy values every p epochs - if s==None: # if s is None, it means the default value of s is used - self.s=1 # assign 1 to s attribute, which means save the model every epoch - self.file_list=None # assign None to file_list attribute, which means do not keep track of saved files - else: - self.s=s-1 # assign s-1 to s attribute, which means save the model every s epochs - self.file_list=[] # assign an empty list to file_list attribute, which means keep track of saved files - if epoch!=None: # if epoch is not None, it means a fixed number of epochs is given - for i in range(epoch): # iterate over each epoch index - t1=time.time() # record the start time of this epoch - self._train(batch,test_batch) # perform one epoch of training using the _train method - if self.stop_flag==True: # if the stop flag is set to True, it means the training has reached the target loss or accuracy values and has been stopped - return # return nothing and end the training - try: - try: - self.nn.ec.assign_add(1) # use the neural network's epoch counter's assign_add method to increase the epoch counter by 1 (for tensorflow platform) - except Exception: # if the neural network does not have an epoch counter or its assign_add method fails - self.nn.ec+=1 # use the normal addition operation to increase the epoch counter by 1 (for pytorch platform) - except Exception: - pass - self.total_epoch+=1 # increase the total epoch number by 1 for all trainings - if epoch%10!=0: - p=epoch-epoch%self.p - p=int(p/self.p) - s=epoch-epoch%self.s - s=int(s/self.s) - else: - p=epoch/(self.p+1) - p=int(p) - s=epoch/(self.s+1) - s=int(s) - if p==0: - p=1 - if s==0: - s=1 - if i%p==0: # if this epoch index is a multiple of p (or p+1) - if self.test_flag==False: # if test data is not used - try: - if self.nn.accuracy!=None: # if the neural network has an accuracy method defined - print('epoch:{0} loss:{1:.6f}'.format(i+1,self.train_loss)) # print the current epoch number and train loss value - if self.acc_flag=='%': # if percentage mode is used for accuracy display - print('epoch:{0} accuracy:{1:.1f}'.format(i+1,self.train_acc*100)) # print the current epoch number and train accuracy value in percentage - else: # if decimal mode is used for accuracy display - print('epoch:{0} accuracy:{1:.6f}'.format(i+1,self.train_acc)) # print the current epoch number and train accuracy value in print('epoch:{0} accuracy:{1:.6f}'.format(i+1,self.train_acc)) # print the current epoch number and train accuracy value in decimal - print() # print an empty line for readability - except Exception: # if the neural network does not have an accuracy method defined or it fails - print('epoch:{0} loss:{1:.6f}'.format(i+1,self.train_loss)) # print the current epoch number and train loss value - print() # print an empty line for readability - else: # if test data is used - try: - if self.nn.accuracy!=None: # if the neural network has an accuracy method defined - print('epoch:{0} loss:{1:.6f},test loss:{2:.6f}'.format(i+1,self.train_loss,self.test_loss)) # print the current epoch number, train loss value and test loss value - if self.acc_flag=='%': # if percentage mode is used for accuracy display - print('epoch:{0} accuracy:{1:.1f},test accuracy:{2:.1f}'.format(i+1,self.train_acc*100,self.test_acc*100)) # print the current epoch number, train accuracy value and test accuracy value in percentage - else: # if decimal mode is used for accuracy display - print('epoch:{0} accuracy:{1:.6f},test accuracy:{2:.6f}'.format(i+1,self.train_acc,self.test_acc)) # print the current epoch number, train accuracy value and test accuracy value in decimal - print() # print an empty line for readability - except Exception: # if the neural network does not have an accuracy method defined or it fails - print('epoch:{0} loss:{1:.6f},test loss:{2:.6f}'.format(i+1,self.train_loss,self.test_loss)) # print the current epoch number, train loss value and test loss value - print() # print an empty line for readability - if save!=None and i%s==0: # if save is not None, it means a file name is given to save the model, and if this epoch index is a multiple of s (or s+1) - self.save(self.total_epoch,one) # save the model using the save method with total epoch number and one flag as inputs - t2=time.time() # record the end time of this epoch - self.time+=(t2-t1) # calculate and update the time elapsed for this training session - else: # if epoch is None, it means an infinite number of epochs is given - i=0 # initialize the epoch index to 0 - while True: # loop indefinitely until stopped by other conditions - t1=time.time() # record the start time of this epoch - self._train(test_batch=test_batch) # perform one epoch of training using the _train method - if self.stop_flag==True: # if the stop flag is set to True, it means the training has reached the target loss or accuracy values and has been stopped - return # return nothing and end the training - i+=1 # increase the epoch index by 1 - try: - try: - self.nn.ec.assign_add(1) # use the neural network's epoch counter's assign_add method to increase the epoch counter by 1 (for tensorflow platform) - except Exception: # if the neural network does not have an epoch counter or its assign_add method fails - self.nn.ec+=1 # use the normal addition operation to increase the epoch counter by 1 (for pytorch platform) - except Exception: - pass - self.total_epoch+=1 # increase the total epoch number by 1 for all trainings - if epoch%10!=0: - p=epoch-epoch%self.p - p=int(p/self.p) - s=epoch-epoch%self.s - s=int(s/self.s) - else: - p=epoch/(self.p+1) - p=int(p) - s=epoch/(self.s+1) - s=int(s) - if p==0: - p=1 - if s==0: - s=1 - if i%p==0: # if this epoch index is a multiple of p (or p+1) - if self.test_flag==False: # if test data is not used - try: - if self.nn.accuracy!=None: # if the neural network has an accuracy method defined - print('epoch:{0} loss:{1:.6f}'.format(i+1,self.train_loss)) # print the current epoch number and train loss value - if self.acc_flag=='%': # if percentage mode is used for accuracy display - print('epoch:{0} accuracy:{1:.1f}'.format(i+1,self.train_acc*100)) # print the current epoch number and train accuracy value in percentage - else: # if decimal mode is used for accuracy display - print('epoch:{0} accuracy:{1:.6f}'.format(i+1,self.train_acc)) # print the current epoch number and train accuracy value in decimal - print() # print an empty line for readability - except Exception: # if the neural network does not have an accuracy method defined or it fails - print('epoch:{0} loss:{1:.6f}'.format(i+1,self.train_loss)) # print the current epoch number and train loss value - print() # print an empty line for readability - else: # if test data is used - try: - if self.nn.accuracy!=None: # if the neural network has an accuracy method defined - print('epoch:{0} loss:{1:.6f},test loss:{2:.6f}'.format(i+1,self.train_loss,self.test_loss)) # print the current epoch number, train loss value and test loss value - if self.acc_flag=='%': # if percentage mode is used for accuracy display - print('epoch:{0} accuracy:{1:.1f},test accuracy:{2:.1f}'.format(i+1,self.train_acc*100,self.test_acc*100)) # print the current epoch number, train accuracy value and test accuracy value in percentage - else: # if decimal mode is used for accuracy display - print('epoch:{0} accuracy:{1:.6f},test accuracy:{2:.6f}'.format(i+1,self.train_acc,self.test_acc)) # print the current epoch number, train accuracy value and test accuracy value in decimal - print() # print an empty line for readability - except Exception: # if the neural network does not have an accuracy method defined or it fails - print('epoch:{0} loss:{1:.6f},test loss:{2:.6f}'.format(i+1,self.train_loss,self.test_loss)) # print the current epoch number, train loss value and test loss value - print() # print an empty line for readability - if save!=None and i%s==0: # if save is not None, it means a file name is given to save the model, and if this epoch index is a multiple of s (or s+1) - self.save(self.total_epoch,one) # save the model using the save method with total epoch number and one flag as inputs - t2=time.time() # record the end time of this epoch - self.time+=(t2-t1) # calculate and update the time elapsed for this training session - if save!=None: # if save is not None, it means a file name is given to save the model - self.save() # save the model using the save method without any inputs (use default values) - self._time=self.time-int(self.time) # get the fractional part of the time elapsed for this training session - if self._time<0.5: # if the fractional part is less than 0.5 - self.time=int(self.time) # round down the time elapsed to integer - else: # if the fractional part is greater than or equal to 0.5 - self.time=int(self.time)+1 # round up the time elapsed to integer - self.total_time+=self.time # calculate and update the total time elapsed for all trainings - if self.test_flag==False: # if test data is not used - print('last loss:{0:.6f}'.format(self.train_loss)) # print the last train loss value - else: # if test data is used - print('last loss:{0:.6f},last test loss:{1:.6f}'.format(self.train_loss,self.test_loss)) # print the last train loss value and test loss value - try: - if self.nn.accuracy!=None: # if the neural network has an accuracy method defined - if self.acc_flag=='%': # if percentage mode is used for accuracy display - if self.test_flag==False: # if test data is not used - print('last accuracy:{0:.1f}'.format(self.train_acc*100)) # print the last train accuracy value in percentage - else: # if test else: # if test data is used - print('last accuracy:{0:.1f},last test accuracy:{1:.1f}'.format(self.train_acc*100,self.test_acc*100)) # print the last train accuracy value and test accuracy value in percentage - else: # if decimal mode is used for accuracy display - if self.test_flag==False: # if test data is not used - print('last accuracy:{0:.6f}'.format(self.train_acc)) # print the last train accuracy value in decimal - else: # if test data is used - print('last accuracy:{0:.6f},last test accuracy:{1:.6f}'.format(self.train_acc,self.test_acc)) # print the last train accuracy value and test accuracy value in decimal - except Exception: # if the neural network does not have an accuracy method defined or it fails - pass - print() # print an empty line for readability - print('time:{0}s'.format(self.time)) # print the time elapsed for this training session - self.training_flag=False # set the training flag to False, which means the training is finished - return # return nothing and end the training - - - def train_ol(self): # a method to perform online learning, which means updating the model with one sample at a time - while True: # loop indefinitely until stopped by other conditions - if self.stop_flag==True: # if the stop flag is set to True, it means the online learning has reached the target loss or accuracy values and has been stopped - return # return nothing and end the online learning - if self.save_flag==True: # if the save flag is set to True, it means a request to save the model has been made - self.save() # save the model using the save method without any inputs (use default values) - self.suspend_func() # check whether a request to suspend the online learning has been made using the suspend_func method - data=self.nn.ol() # get one sample of data and label from the neural network's ol method, which should be defined by the user - if data=='stop': # if the data is 'stop', it means a request to stop the online learning has been made - return # return nothing and end the online learning - elif data=='suspend': # if the data is 'suspend', it means a request to suspend the online learning has been made - self.nn.suspend=True # set the neural network's suspend attribute to True, which means the online learning is suspended - while True: # loop indefinitely until resumed by other conditions - if self.nn.suspend==False: # if the neural network's suspend attribute is set to False, it means a request to resume the online learning has been made - break # break the loop and resume the online learning - continue # continue to the next iteration of online learning - output,loss=self.opt(data[0],data[1]) # perform one optimization step using the opt method and get the output and loss values - loss=loss.numpy() # convert the loss value to numpy array - if len(self.nn.train_loss_list)==self.nn.max_length: # if the train loss list has reached its maximum length, which should be defined by the user - del self.nn.train_loss_list[0] # delete the first element of train loss list - self.nn.train_loss_list.append(loss) # append the loss value to train loss list - try: - train_acc=self.nn.accuracy(output,data[1]) # calculate the accuracy value using the neural network's accuracy method - if len(self.nn.train_acc_list)==self.nn.max_length: # if the train accuracy list has reached its maximum length, which should be defined by the user - del self.nn.train_acc_list[0] # delete the first element of train accuracy list - self.train_acc_list.append(train_acc) # append the accuracy value to train accuracy list - except Exception as e: - try: - if self.nn.accuracy!=None: # if the neural network has an accuracy method defined - raise e # raise the exception - except Exception: - pass - try: - self.nn.c+=1 # increase the neural network's counter by 1, which should be defined by the user to keep track of how many samples have been used for online learning - except Exception: - pass - return # return nothing and end the online learning - - - def test(self,test_data=None,test_labels=None,batch=None): # a method to calculate the test loss and accuracy values using test data and labels - if batch!=None: # if batch mode is used - total_loss=0 # initialize the total loss value for all batches - total_acc=0 # initialize the total accuracy value for all batches - if self.test_dataset!=None: # if a tensorflow or pytorch dataset object is used for test data - for data_batch,labels_batch in self.test_dataset: # iterate over each batch of data and labels from the test dataset - output=self.nn.fp(data_batch) # get the output value using the neural network's forward propagation method - batch_loss=self.nn.loss(output,labels_batch) # get the batch loss value using the neural network's loss function - total_loss+=batch_loss # accumulate the batch loss to total loss - try: - batch_acc=self.nn.accuracy(output,labels_batch) # calculate the batch accuracy using the neural network's accuracy method - total_acc+=batch_acc # accumulate the batch accuracy to total accuracy - except Exception as e: - try: - if self.nn.accuracy!=None: # if the neural network has an accuracy method defined - raise e # raise the exception - except Exception: - pass - else: # if a numpy array is used for test data - total_loss=0 # initialize the total loss value for all batches - total_acc=0 # initialize the total accuracy value for all batches - if type(test_data)==list: # if test data is a list of arrays (for multiple inputs) - batches=int((test_data[0].shape[0]-test_data[0].shape[0]%batch)/batch) # calculate how many batches are needed for test data according to batch size - shape0=test_data[0].shape[0] # get the number of samples in test data - else: # if test data is a single array - batches=int((test_data.shape[0]-test_data.shape[0]%batch)/batch) # calculate how many batches are needed for test data according to batch size - shape0=test_data.shape[0] # get the number of samples in test data - for j in range(batches): # iterate over each batch index - index1=j*batch # calculate the start index of test data for this batch - index2=(j+1)*batch # calculate the end index of test data for this batch - if type(test_data)==list: # if test data is a list of arrays (for multiple inputs) - for i in range(len(test_data)): - data_batch[i]=test_data[i][index1:index2] # get a slice of test data according to the index range for each input array - else: # if test data is a single array - data_batch=test_data[index1:index2] # get a slice of test data according to the index range - if type(test_labels)==list: # if test labels is a list of arrays (for multiple outputs) - for i in range(len(test_labels)): - labels_batch[i]=test_labels[i][index1:index2] # get a slice of test labels according to the index range for each output array - else: # if test labels is a single array - labels_batch=test_labels[index1:index2] # get a slice of test labels according to the index range - output=self.nn.fp(data_batch) # get the output value using the neural network's forward propagation method - batch_loss=self.nn.loss(output,labels_batch) # get the batch loss value using the neural network's loss function - total_loss+=batch_loss # accumulate the batch loss to total loss - try: - batch_acc=self.nn.accuracy(output,labels_batch) # calculate the batch accuracy using the neural network's accuracy method - total_acc+=batch_acc # accumulate the batch accuracy to total accuracy - except Exception as e: - try: - if self.nn.accuracy!=None: # if the neural network has an accuracy method defined - raise e # raise the exception - except Exception: - pass - if shape0%batch!=0: # if there are some samples left in test data that are not enough to form a full batch - batches+=1 # increase the number of batches by 1 to include the remaining samples - index1=batches*batch # calculate the start index of test data for the last batch - index2=batch-(shape0-batches*batch) # calculate how many samples are needed from the beginning of test data to form a full batch - try: - try: - if type(test_data)==list: # if test data is a list of arrays (for multiple if type(test_data)==list: # if test data is a list of arrays (for multiple inputs) - for i in range(len(test_data)): - data_batch[i]=self.platform.concat([test_data[i][index1:],test_data[i][:index2]],0) # concatenate two slices of test data from the end and the beginning to form a batch for each input array - else: # if test data is a single array - data_batch=self.platform.concat([test_data[index1:],test_data[:index2]],0) # concatenate two slices of test data from the end and the beginning to form a batch - if type(test_labels)==list: # if test labels is a list of arrays (for multiple outputs) - for i in range(len(test_labels)): - labels_batch[i]=self.platform.concat([test_labels[i][index1:],test_labels[i][:index2]],0) # concatenate two slices of test labels from the end and the beginning to form a batch for each output array - else: # if test labels is a single array - labels_batch=self.platform.concat([test_labels[index1:],test_labels[:index2]],0) # concatenate two slices of test labels from the end and the beginning to form a batch - except Exception: # if the platform's concat method fails - if type(test_data)==list: # if test data is a list of arrays (for multiple inputs) - for i in range(len(test_data)): - data_batch[i]=np.concatenate([test_data[i][index1:],test_data[i][:index2]],0) # use numpy's concatenate method instead for each input array - else: # if test data is a single array - data_batch=np.concatenate([test_data[index1:],test_data[:index2]],0) # use numpy's concatenate method instead - if type(test_labels)==list: # if test labels is a list of arrays (for multiple outputs) - for i in range(len(test_labels)): - labels_batch[i]=np.concatenate([test_labels[i][index1:],test_labels[i][:index2]],0) # use numpy's concatenate method instead for each output array - else: # if test labels is a single array - labels_batch=np.concatenate([test_labels[index1:],test_labels[:index2]],0) # use numpy's concatenate method instead - except Exception as e: - raise e # raise any other exception - output=self.nn.fp(data_batch) # get the output value using the neural network's forward propagation method - batch_loss=self.nn.loss(output,labels_batch) # get the batch loss value using the neural network's loss function - total_loss+=batch_loss # accumulate the batch loss to total loss - try: - batch_acc=self.nn.accuracy(output,labels_batch) # calculate the batch accuracy using the neural network's accuracy method - total_acc+=batch_acc # accumulate the batch accuracy to total accuracy - except Exception as e: - try: - if self.nn.accuracy!=None: # if the neural network has an accuracy method defined - raise e # raise the exception - except Exception: - pass - test_loss=total_loss.numpy()/batches # convert the total loss value to numpy array and divide it by number of batches to get the average loss value for test data - try: - if self.nn.accuracy!=None: # if the neural network has an accuracy method defined - test_acc=total_acc.numpy()/batches # convert the total accuracy value to numpy array and divide it by number of batches to get the average accuracy value for test data - except Exception: - pass - else: # if batch mode is not used (use all data at once) - output=self.nn.fp(test_data) # get the output value using the neural network's forward propagation method - test_loss=self.nn.loss(output,test_labels) # get the loss value using the neural network's loss function - test_loss=test_loss.numpy() # convert the loss value to numpy array - try: - test_acc=self.nn.accuracy(output,test_labels) # calculate the accuracy value using the neural network's accuracy method - test_acc=test_acc.numpy() # convert the accuracy value to numpy array - except Exception as e: - try: - if self.nn.accuracy!=None: # if the neural network has an accuracy method defined - raise e # raise the exception - except Exception: - pass - try: - if self.nn.accuracy!=None: # if the neural network has an accuracy method defined - return test_loss,test_acc # return the test loss and accuracy values - except Exception: - return test_loss,None # return the test loss value and None - - - def suspend_func(self): # a method to check whether a request to suspend the training or online learning has been made - if self.suspend==True: # if the suspend flag is set to True - if self.save_epoch==None: # if the save epoch number is None, it means no request to save the model has been made - print('Training have suspended.') # print a message to indicate the training or online learning has been suspended - else: - self._save() # save the model using the _save method - while True: # loop indefinitely until resumed by other conditions - if self.suspend==False: # if the suspend flag is set to False, it means a request to resume the training or online learning has been made - print('Training have continued.') # print a message to indicate the training or online learning has been resumed - break # break the loop and resume the training or online learning - return # return nothing for this case - - - def stop_func(self): # a method to check whether the training or online learning has reached the target loss or accuracy values and stop it accordingly - if self.end(): # check whether the target loss or accuracy values have been reached using the end method - self.save(self.total_epoch,True) # save the model using the save method with total epoch number and True flag as inputs - print('\nSystem have stopped training,Neural network have been saved.') # print a message to indicate the training or online learning has been stopped and the model has been saved - self._time=self.time-int(self.time) # get the fractional part of the time elapsed for this training session - if self._time<0.5: # if the fractional part is less than 0.5 - self.time=int(self.time) # round down the time elapsed to integer - else: # if the fractional part is greater than or equal to 0.5 - self.time=int(self.time)+1 # round up the time elapsed to integer - self.total_time+=self.time # calculate and update the total time elapsed for all trainings - print() # print an empty line for readability - print('epoch:{0}'.format(self.total_epoch)) # print the total epoch number for all trainings - if self.test_flag==False: # if test data is not used - print('last loss:{0:.6f}'.format(self.train_loss)) # print the last train loss value - else: # if test data is used - print('last loss:{0:.6f},last test loss:{1:.6f}'.format(self.train_loss,self.test_loss)) # print the last train loss value and test loss value - try: - if self.nn.accuracy!=None: # if the neural network has an accuracy method defined - if self.acc_flag=='%': # if percentage mode is used for accuracy display - if self.test_flag==False: # if test data is not used - print('last accuracy:{0:.1f}'.format(self.train_acc*100)) # print the last train accuracy value in percentage - else: # if test data is used - print('last accuracy:{0:.1f},last test accuracy:{1:.1f}'.format(self.train_acc*100,self.test_acc*100)) # print the last train accuracy value and test accuracy value in percentage - else: # if decimal mode is used for accuracy display - if self.test_flag==False: # if test data is not used - print('last accuracy:{0:.6f}'.format(self.train_acc)) # print the last train accuracy value in decimal - else: # if test data is used - print('last accuracy:{0:.6f},last test accuracy:{1:.6f}'.format(self.train_acc,self.test_acc)) # print the last train accuracy value and test accuracy value in decimal - except Exception: # if the neural network does not have an accuracy method defined or it fails - pass - print() # print an empty line for readability - print('time:{0}s'.format(self.total_time)) # print the total time elapsed for all trainings - self.stop_flag=True # set the stop flag to True, which means the training or online learning has been stopped - return True # return True to indicate that the training or online learning has been stopped - return False # return False to indicate that the training or online learning has not been stopped - - - def stop_func_(self): # a method to check whether a request to stop the training or online learning has been made or the target loss or accuracy values have been reached - if self.stop==True: # if the stop flag is set to True, it means a request to stop the training or online learning has been made - if self.stop_flag==True or self.stop_func(): # if the stop flag is already True, it means the target loss or accuracy values have been reached, or check whether the target loss or accuracy values have been reached using the stop_func method - return True # return True to indicate that the training or online learning has been stopped - return False # return False to indicate that the training or online learning has not been stopped - - - def visualize_train(self): # a method to visualize the train loss and accuracy values using matplotlib.pyplot - print() # print an empty line for readability - plt.figure(1) # create a new figure with number 1 - plt.plot(np.arange(self.total_epoch),self.train_loss_list) # plot the train loss list with x-axis as the epoch number - plt.title('train loss') # set the title of the plot to 'train loss' - plt.xlabel('epoch') # set the x-axis label to 'epoch' - plt.ylabel('loss') # set the y-axis label to 'loss' - print('train loss:{0:.6f}'.format(self.train_loss)) # print the last train loss value - try: - if self.nn.accuracy!=None: # if the neural network has an accuracy method defined - plt.figure(2) # create a new figure with number 2 - plt.plot(np.arange(self.total_epoch),self.train_acc_list) # plot the train accuracy list with x-axis as the epoch number - plt.title('train acc') # set the title of the plot to 'train acc' - plt.xlabel('epoch') # set the x-axis label to 'epoch' - plt.ylabel('acc') # set the y-axis label to 'acc' - if self.acc_flag=='%': # if percentage mode is used for accuracy display - print('train acc:{0:.1f}'.format(self.train_acc*100)) # print the last train accuracy value in percentage - else: # if decimal mode is used for accuracy display - print('train acc:{0:.6f}'.format(self.train_acc)) # print the last train accuracy value in decimal - except Exception: # if the neural network does not have an accuracy method defined or it fails - pass - return # return nothing for this case diff --git a/Note 7.0 pv documentation/DL/reduced kernel/process/kernel_reduced.py b/Note 7.0 pv documentation/DL/reduced kernel/process/kernel_reduced.py deleted file mode 100644 index 6561c7f3..00000000 --- a/Note 7.0 pv documentation/DL/reduced kernel/process/kernel_reduced.py +++ /dev/null @@ -1,637 +0,0 @@ -import tensorflow as tf -from tensorflow.python.ops import state_ops -from tensorflow.python.util import nest -from multiprocessing import Value,Array -import numpy as np -import matplotlib.pyplot as plt - - -class kernel: - def __init__(self,nn=None): - self.nn=nn # the neural network object - try: - self.nn.km=1 # a flag to indicate the kernel mode - except Exception: - pass - self.PO=None # the order of the optimizer - self.process=None # the number of processes - self.train_ds=None # the training dataset - self.data_segment_flag=False # a flag to indicate whether to segment the data - self.batches=None # the number of batches - self.buffer_size=None # the buffer size for shuffling the data - self.priority_flag=False # a flag to indicate whether to use priority for optimization - self.priority_p=0 # the priority parameter - self.max_opt=None # the maximum number of optimization steps - self.epoch=None # the number of epochs - self.epoch_counter=0 # the epoch counter - self.stop=False # a flag to indicate whether to stop the training - self.stop_flag=False # a flag to indicate whether to stop the training by condition - self.save_flag=False # a flag to indicate whether to save the model - self.batch=None # the batch size - self.end_loss=None # the end condition for training loss - self.end_acc=None # the end condition for training accuracy - self.end_test_loss=None # the end condition for test loss - self.end_test_acc=None # the end condition for test accuracy - self.acc_flag='%' # the format for displaying accuracy - self.opt_counter=None # the counter for optimization steps - self.train_loss=0 # the training loss - self.train_acc=0 # the training accuracy - self.train_loss_list=[] # the list of training loss values - self.train_acc_list=[] # the list of training accuracy values - self.test_loss=0 # the test loss - self.test_acc=0 # the test accuracy - self.test_loss_list=[] # the list of test loss values - self.test_acc_list=[] # the list of test accuracy values - self.test_flag=False # a flag to indicate whether to use test data - self.total_epoch=0 # the total number of epochs - - def data(self,train_data=None,train_labels=None,test_data=None,test_labels=None,train_dataset=None,test_dataset=None): - if type(self.nn.param[0])!=list: - self.train_data=train_data.astype(self.nn.param[0].dtype.name) # convert the train data type to match the model parameter type - self.train_labels=train_labels.astype(self.nn.param[0].dtype.name) # convert the train labels type to match the model parameter type - else: - self.train_data=train_data.astype(self.nn.param[0][0].dtype.name) # convert the train data type to match the model parameter type - self.train_labels=train_labels.astype(self.nn.param[0][0].dtype.name) # convert the train labels type to match the model parameter type - self.train_dataset=train_dataset # set the train dataset object - if test_data is not None: - self.test_data=test_data # set the test data array - self.test_labels=test_labels # set the test labels array - self.test_flag=True # set the test flag to True - self.test_dataset=test_dataset # set the test dataset object - self.batch_counter=np.zeros(self.process,dtype=np.int32) # initialize an array to count batches for each process - if type(self.nn.param[0])!=list: # initialize an array to accumulate loss for each process - self.total_loss=np.zeros(self.process,dtype=self.nn.param[0].dtype.name) - else: - self.total_loss=np.zeros(self.process,dtype=self.nn.param[0][0].dtype.name) - try: - if self.nn.accuracy!=None: - if type(self.nn.param[0])!=list: # initialize an array to accumulate accuracy for each process - self.total_acc=np.zeros(self.process,dtype=self.nn.param[0].dtype.name) - else: - self.total_acc=np.zeros(self.process,dtype=self.nn.param[0][0].dtype.name) - except Exception: - pass - if self.priority_flag==True: - self.opt_counter=np.zeros(self.process,dtype=np.int32) # initialize an array to count optimization steps for each process - if self.train_dataset==None: - if type(self.train_data)==list: - self.shape0=train_data[0].shape[0] # get the number of samples in the first train data array - self.batches=int((self.shape0-self.shape0%self.batch)/self.batch) # calculate the number of batches - if self.shape0%self.batch!=0: - self.batches+=1 # add one more batch if there are remaining samples - else: - self.shape0=train_data.shape[0] # get the number of samples in the train data array - self.batches=int((self.shape0-self.shape0%self.batch)/self.batch) # calculate the number of batches - if self.shape0%self.batch!=0: - self.batches+=1 # add one more batch if there are remaining samples - if self.data_segment_flag==True: - self.train_data,self.train_labels=self.segment_data() # segment the train data and labels according to the number of processes - return - - - def segment_data(self): - if len(self.train_data)!=self.process: # check if the train data is already segmented - segments=int((len(self.train_data)-len(self.train_data)%self.process)/self.process) # calculate the number of samples for each segment - for i in range(self.process): # loop over the processes - index1=i*segments # get the start index of the segment - index2=(i+1)*segments # get the end index of the segment - if i==0: # for the first process - data=np.expand_dims(self.train_data[index1:index2],axis=0) # create a new dimension for the segment and assign it to data - labels=np.expand_dims(self.train_labels[index1:index2],axis=0) # create a new dimension for the segment and assign it to labels - else: # for other processes - data=np.concatenate((data,np.expand_dims(self.train_data[index1:index2],axis=0))) # concatenate the segment to data along the new dimension - labels=np.concatenate((labels,np.expand_dims(self.train_labels[index1:index2],axis=0))) # concatenate the segment to labels along the new dimension - if len(data)%self.process!=0: # check if there are remaining samples that are not segmented - segments+=1 # increase the number of samples for each segment by one - index1=segments*self.process # get the start index of the remaining samples - index2=self.process-(len(self.train_data)-segments*self.process) # get the number of processes that need to be filled with extra samples - data=np.concatenate((data,np.expand_dims(self.train_data[index1:index2],axis=0))) # concatenate the remaining samples to data along the new dimension - labels=np.concatenate((labels,np.expand_dims(self.train_labels[index1:index2],axis=0))) # concatenate the remaining samples to labels along the new dimension - return data,labels # return the segmented data and labels - - - def init(self,manager): - self.epoch_counter=Value('i',self.epoch_counter) # create a shared value for epoch counter - self.batch_counter=Array('i',self.batch_counter) # create a shared array for batch counter - self.total_loss=Array('f',self.total_loss) # create a shared array for total loss - self.total_epoch=Value('i',self.total_epoch) # create a shared value for total epoch - self.train_loss=Value('f',self.train_loss) # create a shared value for train loss - self.train_loss_list=manager.list(self.train_loss_list) # create a shared list for train loss values - self.priority_p=Value('i',self.priority_p) # create a shared value for priority parameter - if self.test_flag==True: - self.test_loss=Value('f',self.test_loss) # create a shared value for test loss - self.test_loss_list=manager.list(self.test_loss_list) # create a shared list for test loss values - try: - if self.nn.accuracy!=None: - self.total_acc=Array('f',self.total_acc) # create a shared array for total accuracy - self.train_acc=Value('f',self.train_acc) # create a shared value for train accuracy - self.train_acc_list=manager.list(self.train_acc_list) # create a shared list for train accuracy values - if self.test_flag==True: - self.test_acc=Value('f',self.test_acc) # create a shared value for test accuracy - self.test_acc_list=manager.list(self.test_acc_list) # create a shared list for test accuracy values - except Exception: - pass - if self.priority_flag==True: - self.opt_counter=Array('i',self.opt_counter) # create a shared array for optimization counter - try: - if self.nn.attenuate!=None: - self.nn.opt_counter=manager.list([self.nn.opt_counter]) # create a shared list for the neural network's optimization counter - except Exception: - pass - try: - self.nn.ec=manager.list([self.nn.ec]) # create a shared list for the neural network's epoch counter - except Exception: - pass - try: - self.nn.bc=manager.list([self.nn.bc]) # create a shared list for the neural network's batch counter - except Exception: - pass - self.stop_flag=Value('b',self.stop_flag) # create a shared value for stop flag - self.save_flag=Value('b',self.save_flag) # create a shared value for save flag - self.param=manager.dict() # create a shared dictionary for parameters - self.param[7]=self.nn.param # assign the neural network's parameters to the dictionary - return - - - def end(self): - if self.end_loss!=None and len(self.train_loss_list)!=0 and self.train_loss_list[-1]self.end_acc: # check if the train accuracy is higher than the end condition - return True - elif self.end_loss!=None and len(self.train_loss_list)!=0 and self.end_acc!=None and self.train_loss_list[-1]self.end_acc: # check if both the train loss and accuracy meet the end condition - return True - elif self.end_test_loss!=None and len(self.test_loss_list)!=0 and self.test_loss_list[-1]self.end_test_acc: # check if the test accuracy is higher than the end condition - return True - elif self.end_test_loss!=None and len(self.test_loss_list)!=0 and self.end_test_acc!=None and self.test_loss_list[-1]self.end_test_acc: # check if both the test loss and accuracy meet the end condition - return True - - - @tf.function - def opt_p(self,data,labels,p,lock,g_lock=None): - try: - try: - if self.nn.GradientTape!=None: # check if the neural network has its own gradient tape function - tape,output,loss=self.nn.GradientTape(data,labels,p) # use the neural network's gradient tape function to get the tape, output and loss - except Exception: - with tf.GradientTape(persistent=True) as tape: # use the default gradient tape function - try: - try: - output=self.nn.fp(data) # use the neural network's forward propagation function to get the output - loss=self.nn.loss(output,labels) # use the neural network's loss function to get the loss - except Exception: - output,loss=self.nn.fp(data,labels) # use the neural network's forward propagation function to get the output and loss - except Exception: - try: - output=self.nn.fp(data,p) # use the neural network's forward propagation function to get the output with the process number - loss=self.nn.loss(output,labels) # use the neural network's loss function to get the loss - except Exception: - output,loss=self.nn.fp(data,labels,p) # use the neural network's forward propagation function to get the output and loss with the process number - except Exception as e: - raise e - if self.PO==1: # check if the optimizer order is 1 (lock before gradient calculation) - if self.priority_flag==True and self.priority_p.value!=-1: # check if the priority flag is True and the priority parameter is not -1 - while True: - if p==self.priority_p.value: # check if the process number matches the priority parameter - break - else: - continue - lock[0].acquire() # acquire the first lock - if self.stop_func_(lock[0]): # check if the stop condition is met - return None,0 - try: - try: - try: - gradient=self.nn.gradient(tape,loss) # use the neural network's gradient function to get the gradient - except Exception: - gradient=self.nn.gradient(tape,loss,self.param[7]) # use the neural network's gradient function to get the gradient with the parameters - except Exception: - gradient=tape.gradient(loss,self.nn.param) # use the default gradient function to get the gradient - except Exception as e: - raise e - try: - gradient=self.nn.attenuate(gradient,self.nn.opt_counter,p) # use the neural network's attenuate function to modify the gradient - except Exception as e: - try: - if self.nn.attenuate!=None: # check if the neural network has an attenuate function - raise e - except Exception: - pass - try: - try: - param=self.nn.opt(gradient) # use the neural network's optimizer function to update the parameters - except Exception: - param=self.nn.opt(gradient,p) # use the neural network's optimizer function to update the parameters with the process number - except Exception as e: - raise e - lock[0].release() # release the first lock - elif self.PO==2: # check if the optimizer order is 2 (lock after gradient calculation) - g_lock.acquire() # acquire the global lock - if self.stop_func_(g_lock): # check if the stop condition is met - return None,0 - try: - try: - try: - gradient=self.nn.gradient(tape,loss) # use the neural network's gradient function to get the gradient - except Exception: - gradient=self.nn.gradient(tape,loss,self.param[7]) # use the neural network's gradient function to get the gradient with the parameters - except Exception: - gradient=tape.gradient(loss,self.nn.param) # use the default gradient function to get the gradient - except Exception as e: - raise e - g_lock.release() # release the global lock - if self.priority_flag==True and self.priority_p.value!=-1: # check if the priority flag is True and the priority parameter is not -1 - while True: - if p==self.priority_p.value: # check if the process number matches the priority parameter - break - else: - continue - lock[0].acquire() # acquire the first lock - if self.stop_func_(lock[0]): # check if the stop condition is met - return None,0 - try: - gradient=self.nn.attenuate(gradient,self.nn.opt_counter,p) # use the neural network's attenuate function to modify the gradient - except Exception as e: - try: - if self.nn.attenuate!=None: # check if the neural network has an attenuate function - raise e - except Exception: - pass - try: - try: - param=self.nn.opt(gradient) # use the neural network's optimizer function to update the parameters - except Exception: - param=self.nn.opt(gradient,p) # use the neural network's optimizer function to update the parameters with the process number - except Exception as e: - raise e - lock[0].release() # release the first lock - elif self.PO==3: # check if the optimizer order is 3 (no lock) - if self.priority_flag==True and self.priority_p.value!=-1: # check if the priority flag is True and the priority parameter is not -1 - while True: - if p==self.priority_p.value: # check if the process number matches the priority parameter - break - else: - continue - if self.stop_func_(): # check if the stop condition is met - return None,0 - try: - try: - try: - gradient=self.nn.gradient(tape,loss) # use the neural network's gradient function to get the gradient - except Exception: - gradient=self.nn.gradient(tape,loss,self.param[7]) # use the neural network's gradient function to get the gradient with the parameters - except Exception: - gradient=tape.gradient(loss,self.nn.param) # use the default gradient function to get the gradient - except Exception as e: - raise e - try: - gradient=self.nn.attenuate(gradient,self.nn.opt_counter,p) # use the neural network's attenuate function to modify the gradient - except Exception as e: - try: - if self.nn.attenuate!=None: # check if the neural network has an attenuate function - raise e - except Exception: - pass - try: - try: - param=self.nn.opt(gradient) # use the neural network's optimizer function to update the parameters - except Exception: - param=self.nn.opt(gradient,p) # use the neural network's optimizer function to update the parameters with the process number - except Exception as e: - raise e - return output,loss,param # return output, loss and parameters - - - def opt(self,data,labels,p,lock,g_lock): - if self.PO==2: # check if the optimizer order is 2 (lock after gradient calculation) - if type(g_lock)!=list: # check if g_lock is not a list - pass - elif len(g_lock)==self.process: # check if g_lock has same length as process number - ln=p # assign process number to ln (local number) - g_lock=g_lock[ln] # assign g_lock element at ln index to g_lock - else: - ln=int(np.random.choice(len(g_lock))) # assign a random integer from g_lock length to ln (local number) - g_lock=g_lock[ln] # assign g_lock element at ln index to g_lock - output,loss,param=self.opt_p(data,labels,p,lock,g_lock) # call the opt_p function to get output, loss and parameters - else: - output,loss,param=self.opt_p(data,labels,p,lock) # call the opt_p function to get output, loss and parameters without g_lock - return output,loss,param # return output, loss and parameters - - - def update_nn_param(self,param=None): - if param==None: # check if param is None - parameter_flat=nest.flatten(self.nn.param) # flatten the neural network's parameters - parameter7_flat=nest.flatten(self.param[7]) # flatten the kernel's parameters - else: - parameter_flat=nest.flatten(self.nn.param) # flatten the neural network's parameters - parameter7_flat=nest.flatten(param) # flatten the given param - for i in range(len(parameter_flat)): # loop over the flattened parameters - if param==None: # check if param is None - state_ops.assign(parameter_flat[i],parameter7_flat[i]) # assign the kernel's parameters to the neural network's parameters - else: - state_ops.assign(parameter_flat[i],parameter7_flat[i]) # assign the given param to the neural network's parameters - self.nn.param=nest.pack_sequence_as(self.nn.param,parameter_flat) # pack the flattened parameters back to the neural network's parameters - self.param[7]=nest.pack_sequence_as(self.param[7],parameter7_flat) # pack the flattened parameters back to the kernel's parameters - return - - - def train7(self,train_ds,p,test_batch,lock,g_lock): - while True: # loop until break - for data_batch,labels_batch in train_ds: # loop over the train dataset batches - try: - data_batch,labels_batch=self.nn.data_func(data_batch,labels_batch) # use the neural network's data function to process data and labels - except Exception as e: - try: - if self.nn.data_func!=None: # check if the neural network has a data function - raise e - except Exception: - pass - if self.priority_flag==True: # check if the priority flag is True - self.priority_p.value=np.argmax(self.opt_counter) # assign the index of the maximum value in opt_counter to priority parameter - if self.max_opt!=None and self.opt_counter[self.priority_p.value]>=self.max_opt: # check if max_opt is not None and opt_counter at priority parameter index is greater than or equal to max_opt - self.priority_p.value=int(self.priority_p.value) # convert priority parameter to integer type - elif self.max_opt==None: # check if max_opt is None - self.priority_p.value=int(self.priority_p.value) # convert priority parameter to integer type - else: - self.priority_p.value=-1 # assign -1 to priority parameter - if self.priority_flag==True: # check if the priority flag is True - self.opt_counter[p]=0 # assign zero to opt_counter at process number - try: - opt_counter=self.nn.opt_counter[0] # get the neural network's optimization counter - opt_counter.scatter_update(tf.IndexedSlices(0,p)) # update opt_counter with zero at process number - self.nn.opt_counter[0]=opt_counter # assign opt_counter back to neural network's optimization counter - except Exception as e: - try: - if self.nn.attenuate!=None: # check if the neural network has an attenuate function - raise e - except Exception: - pass - output,batch_loss,param=self.opt(data_batch,labels_batch,p,lock,g_lock) # call the opt function to get output, batch loss and parameters - self.param[7]=param # assign param to kernel's parameters - if self.priority_flag==True: # check if the priority flag is True - opt_counter=np.frombuffer(self.opt_counter.get_obj(),dtype='i') # get a numpy array from opt_counter shared array object - opt_counter+=1 # increment opt_counter by one for each element - try: - opt_counter=self.nn.opt_counter[0] # get the neural network's optimization counter - opt_counter.assign(opt_counter+1) # increment opt_counter by one - self.nn.opt_counter[0]=opt_counter # assign opt_counter back to neural network's optimization counter - except Exception as e: - try: - if self.nn.attenuate!=None: # check if the neural network has an attenuate function - raise e - except Exception: - pass - try: - bc=self.nn.bc[0] # get the neural network's batch counter - bc.assign_add(1) # increment bc by one - self.nn.bc[0]=bc # assign bc back to neural network's batch counter - except Exception: - pass - try: - batch_acc=self.nn.accuracy(output,labels_batch) # use the neural network's accuracy function to get the batch accuracy - except Exception as e: - try: - if self.nn.accuracy!=None: # check if the neural network has an accuracy function - raise e - except Exception: - pass - try: - if self.nn.accuracy!=None: # check if the neural network has an accuracy function - self.total_loss[p]+=batch_loss # accumulate batch loss to total loss at process number - self.total_acc[p]+=batch_acc # accumulate batch accuracy to total accuracy at process number - except Exception: - self.total_loss[p]+=batch_loss # accumulate batch loss to total loss at process number - self.batch_counter[p]+=1 # increment batch counter at process number - if self.PO==1 or self.PO==2: # check if the optimizer order is 1 or 2 (lock before or after gradient calculation) - lock[1].acquire() # acquire the second lock - elif lock!=None: # check if lock is not None - lock.acquire() # acquire the lock - batches=np.sum(self.batch_counter) # sum up the batch counter for all processes - if batches>=self.batches: # check if the number of batches reaches the total number of batches - batch_counter=np.frombuffer(self.batch_counter.get_obj(),dtype='i') # get a numpy array from batch counter shared array object - batch_counter*=0 # reset batch counter to zero for each element - loss=np.sum(self.total_loss)/batches # calculate the average loss for all batches - try: - if self.nn.accuracy!=None: # check if the neural network has an accuracy function - train_acc=np.sum(self.total_acc)/batches # calculate the average accuracy for all batches - except Exception: - pass - self.total_epoch.value+=1 # increment total epoch by one - self.train_loss.value=loss # assign loss to train loss - self.train_loss_list.append(loss) # append loss to train loss list - try: - if self.nn.accuracy!=None: # check if the neural network has an accuracy function - self.train_acc.value=train_acc # assign train_acc to train accuracy - self.train_acc_list.append(train_acc) # append train_acc to train accuracy list - except Exception: - pass - if self.test_flag==True: # check if the test flag is True - self.test_loss.value,self.test_acc.value=self.test(self.test_data,self.test_labels,test_batch,p) # call the test function to get test loss and accuracy - self.test_loss_list.append(self.test_loss.value) # append test loss to test loss list - try: - if self.nn.accuracy!=None: # check if the neural network has an accuracy function - self.test_acc_list.append(self.test_acc.value) # append test accuracy to test accuracy list - except Exception: - pass - self.print_save() # call the print_save function to print and save the results - self.epoch_counter.value+=1 # increment epoch counter by one - try: - ec=self.nn.ec[0] # get the neural network's epoch counter - ec.assign_add(1) # increment ec by one - self.nn.ec[0]=ec # assign ec back to neural network's epoch counter - except Exception: - pass - total_loss=np.frombuffer(self.total_loss.get_obj(),dtype='f') # get a numpy array from total loss shared array object - total_loss*=0 # reset total loss to zero for each element - try: - total_acc=np.frombuffer(self.total_acc.get_obj(),dtype='f') # get a numpy array from total accuracy shared array object - total_acc*=0 # reset total accuracy to zero for each element - except Exception as e: - try: - if self.nn.accuracy!=None: # check if the neural network has an accuracy function - raise e - except Exception: - pass - if self.PO==1 or self.PO==2: # check if the optimizer order is 1 or 2 (lock before or after gradient calculation) - lock[1].release() # release the second lock - elif lock!=None: # check if lock is not None - lock.release() # release the lock - if self.epoch_counter.value>=self.epoch: # check if the epoch counter reaches the epoch number - self.param[7]=param # assign param to kernel's parameters - return - - - def train(self,p,lock=None,g_lock=None,test_batch=None): - if self.epoch!=None: # check if epoch is not None - if self.train_dataset!=None: # check if train dataset is not None - train_ds=self.train_dataset # assign train dataset to train_ds - else: - if self.data_segment_flag==True: # check if data segment flag is True - train_ds=tf.data.Dataset.from_tensor_slices((self.train_data[p],self.train_labels[p])).batch(self.batch) # create a dataset from tensor slices of segmented data and labels and batch them - elif self.buffer_size!=None: # check if buffer size is not None - train_ds=tf.data.Dataset.from_tensor_slices((self.train_data,self.train_labels)).shuffle(self.buffer_size).batch(self.batch) # create a dataset from tensor slices of data and labels and shuffle and batch them with buffer size - else: - train_ds=tf.data.Dataset.from_tensor_slices((self.train_data,self.train_labels)).batch(self.batch) # create a dataset from tensor slices of data and labels and batch them - self.train7(train_ds,p,test_batch,lock,g_lock) # call the train7 function with train_ds, process number, test batch, lock and g_lock - return - - - def test(self,test_data=None,test_labels=None,batch=None,p=None): - if type(self.nn.param[0])!=list: - test_data=test_data.astype(self.nn.param[0].dtype.name) # convert the test data type to match the model parameter type - test_labels=test_labels.astype(self.nn.param[0].dtype.name) # convert the test labels type to match the model parameter type - else: - test_data=test_data.astype(self.nn.param[0][0].dtype.name) # convert the test data type to match the model parameter type - test_labels=test_labels.astype(self.nn.param[0][0].dtype.name) # convert the test labels type to match the model parameter type - if batch!=None: # check if batch is not None - total_loss=0 # initialize total loss to zero - total_acc=0 # initialize total accuracy to zero - if self.test_dataset!=None: # check if test dataset is not None - for data_batch,labels_batch in self.test_dataset: # loop over the test dataset batches - try: - try: - output=self.nn.fp(data_batch) # use the neural network's forward propagation function to get the output - except Exception: - output=self.nn.fp(data_batch,p) # use the neural network's forward propagation function to get the output with the process number - except Exception as e: - raise e - batch_loss=self.nn.loss(output,labels_batch) # use the neural network's loss function to get the batch loss - total_loss+=batch_loss # accumulate batch loss to total loss - try: - batch_acc=self.nn.accuracy(output,labels_batch) # use the neural network's accuracy function to get the batch accuracy - total_acc+=batch_acc # accumulate batch accuracy to total accuracy - except Exception as e: - try: - if self.nn.accuracy!=None: # check if the neural network has an accuracy function - raise e - except Exception: - pass - else: # if test dataset is None - total_loss=0 # initialize total loss to zero - total_acc=0 # initialize total accuracy to zero - batches=int((test_data.shape[0]-test_data.shape[0]%batch)/batch) # calculate the number of batches - shape0=test_data.shape[0] # get the number of samples in the test data array - for j in range(batches): # loop over the batches - index1=j*batch # get the start index of the batch - index2=(j+1)*batch # get the end index of the batch - data_batch=test_data[index1:index2] # get the data batch from test data array - if type(test_labels)==list: # check if test labels is a list - for i in range(len(test_labels)): - labels_batch[i]=test_labels[i][index1:index2] # get the labels batch from test labels list - else: - labels_batch=test_labels[index1:index2] # get the labels batch from test labels array - try: - try: - output=self.nn.fp(data_batch) # use the neural network's forward propagation function to get the output - except Exception: - output=self.nn.fp(data_batch,p) # use the neural network's forward propagation function to get the output with the process number - except Exception as e: - raise e - batch_loss=self.nn.loss(output,labels_batch) # use the neural network's loss function to get the batch loss - total_loss+=batch_loss # accumulate batch loss to total loss - try: - batch_acc=self.nn.accuracy(output,labels_batch) # use the neural network's accuracy function to get the batch accuracy - total_acc+=batch_acc # accumulate batch accuracy to total accuracy - except Exception as e: - try: - if self.nn.accuracy!=None: # check if the neural network has an accuracy function - raise e - except Exception: - pass - if shape0%batch!=0: # check if there are remaining samples that are not in a batch - batches+=1 # increment batches by one - index1=batches*batch # get the start index of the remaining samples - index2=batch-(shape0-batches*batch) # get the number of samples that need to be filled with extra samples - data_batch=tf.concat([test_data[index1:],test_data[:index2]],0) # concatenate the remaining samples and extra samples to form a data batch - labels_batch=tf.concat([test_labels[index1:],test_labels[:index2]],0) # concatenate the remaining labels and extra labels to form a labels batch - try: - try: - output=self.nn.fp(data_batch) # use the neural network's forward propagation function to get the output - except Exception: - output=self.nn.fp(data_batch,p) # use the neural network's forward propagation function to get the output with the process number - except Exception as e: - raise e - batch_loss=self.nn.loss(output,labels_batch) # use the neural network's loss function to get the batch loss - total_loss+=batch_loss # accumulate batch loss to total loss - try: - batch_acc=self.nn.accuracy(output,labels_batch) # use the neural network's accuracy function to get the batch accuracy - total_acc+=batch_acc # accumulate batch accuracy to total accuracy - except Exception as e: - try: - if self.nn.accuracy!=None: # check if the neural network has an accuracy function - raise e - except Exception: - pass - test_loss=total_loss.numpy()/batches # calculate the average test loss for all batches - try: - if self.nn.accuracy!=None: # check if the neural network has an accuracy function - test_acc=total_acc.numpy()/batches # calculate the average test accuracy for all batches - except Exception: - pass - else: # if batch is None - try: - try: - output=self.nn.fp(test_data) # use the neural network's forward propagation function to get the output - except Exception: - output=self.nn.fp(test_data,p) # use the neural network's forward propagation function to get the output with the process number - except Exception as e: - raise e - test_loss=self.nn.loss(output,test_labels) # use the neural network's loss function to get the test loss - test_loss=test_loss.numpy() # convert test loss to numpy array - try: - test_acc=self.nn.accuracy(output,test_labels) # use the neural network's accuracy function to get the test accuracy - test_acc=test_acc.numpy() # convert test accuracy to numpy array - except Exception as e: - try: - if self.nn.accuracy!=None: # check if the neural network has an accuracy function - raise e - except Exception: - pass - try: - if self.nn.accuracy!=None: # check if the neural network has an accuracy function - return test_loss,test_acc # return test loss and accuracy - except Exception: - return test_loss,None # return test loss and None - - - def stop_func(self): - if self.end(): # check if any of the end conditions is met - self.save(self.total_epoch.value,True) # save the model with total epoch and True flag - self.save_flag.value=True # set save flag to True - self.stop_flag.value=True # set stop flag to True - return True - return False - - - def stop_func_(self,lock=None): - if self.stop==True: # check if stop is True - if self.stop_flag.value==True or self.stop_func(): # check if the stop flag is True or the stop function returns True - if self.PO!=3: # check if the optimizer order is not 3 (no lock) - lock.release() # release the lock - return True - return False - - - def visualize_train(self): - print() # print a blank line - plt.figure(1) # create a new figure - plt.plot(np.arange(self.total_epoch.value),self.train_loss_list) # plot the train loss list against the total epoch - plt.title('train loss') # set the title of the figure - plt.xlabel('epoch') # set the x-axis label of the figure - plt.ylabel('loss') # set the y-axis label of the figure - print('train loss:{0:.6f}'.format(self.train_loss.value)) # print the train loss value with six decimal places - try: - if self.nn.accuracy!=None: # check if the neural network has an accuracy function - plt.figure(2) # create a new figure - plt.plot(np.arange(self.total_epoch.value),self.train_acc_list) # plot the train accuracy list against the total epoch - plt.title('train acc') # set the title of the figure - plt.xlabel('epoch') # set the x-axis label of the figure - plt.ylabel('acc') # set the y-axis label of the figure - if self.acc_flag=='%': # check if the accuracy format is percentage - print('train acc:{0:.1f}'.format(self.train_acc.value*100)) # print the train accuracy value with one decimal place and percentage sign - else: - print('train acc:{0:.6f}'.format(self.train_acc.value)) # print the train accuracy value with six decimal places - except Exception: - pass - return diff --git a/Note 7.0 pv documentation/RL/kernel.txt b/Note 7.0 pv documentation/RL/kernel.txt deleted file mode 100644 index 6f879b37..00000000 --- a/Note 7.0 pv documentation/RL/kernel.txt +++ /dev/null @@ -1,10 +0,0 @@ -Pool network: -Pool network use multiprocessing parallel and random add episode in pool,which would make data being uncorrelated in pool, -then pools would be used parallel training agent. -Each process have a pool. - -Support stop mechanism. -Support prioritized replay. -Support gradient attenuation. -Support memory protection mechanism. -Support training priority and memory priority. diff --git a/Note 7.0 pv documentation/RL/neural network/pytorch/DDPG.py b/Note 7.0 pv documentation/RL/neural network/pytorch/DDPG.py deleted file mode 100644 index 04953dd7..00000000 --- a/Note 7.0 pv documentation/RL/neural network/pytorch/DDPG.py +++ /dev/null @@ -1,101 +0,0 @@ -import torch -import torch.nn.functional as F -import numpy as np -import gym - - -class actor(torch.nn.Module): - def __init__(self,state_dim,hidden_dim,action_dim,action_bound): - super(actor,self).__init__() - self.fc1=torch.nn.Linear(state_dim,hidden_dim) - self.fc2=torch.nn.Linear(hidden_dim,action_dim) - self.action_bound=action_bound - - - def forward(self,x): - x=F.relu(self.fc1(x)) - return torch.tanh(self.fc2(x))*self.action_bound - - -class critic(torch.nn.Module): - def __init__(self,state_dim,hidden_dim,action_dim): - super(critic,self).__init__() - self.fc1=torch.nn.Linear(state_dim+action_dim,hidden_dim) - self.fc2=torch.nn.Linear(hidden_dim,1) - - - def forward(self,x,a): - cat=torch.cat([x,a],dim=1) - x=F.relu(self.fc1(cat)) - return self.fc2(x) - - -class DDPG: - def __init__(self,hidden_dim,sigma,gamma,tau,actor_lr,critic_lr): - if torch.cuda.is_available(): - self.device=torch.device('cuda') - else: - self.device=torch.device('cpu') - self.genv=gym.make('Pendulum-v0') - state_dim=self.genv.observation_space.shape[0] - action_dim=self.genv.action_space.shape[0] - action_bound=self.genv.action_space.high[0] - self.actor=actor(state_dim,hidden_dim,action_dim,action_bound).to(self.device) - self.critic=critic(state_dim,hidden_dim,action_dim).to(self.device) - self.target_actor=actor(state_dim,hidden_dim,action_dim,action_bound).to(self.device) - self.target_critic=critic(state_dim,hidden_dim,action_dim).to(self.device) - self.target_actor.load_state_dict(self.actor.state_dict()) - self.target_critic.load_state_dict(self.critic.state_dict()) - self.sigma=sigma - self.gamma=gamma - self.tau=tau - self.actor_opt=torch.optim.Adam(self.actor.parameters(),lr=actor_lr) - self.critic_opt=torch.optim.Adam(self.critic.parameters(),lr=critic_lr) - - - def noise(self): - return np.random.normal(scale=self.sigma) - - - def env(self,a=None,initial=None): - if initial==True: - state=self.genv.reset(seed=0) - return state - else: - next_state,reward,done,_=self.genv.step(a) - return next_state,reward,done - - - def loss(self,s,a,next_s,r,d): - s=torch.tensor(s,dtype=torch.float).to(self.device) - a=torch.tensor(a,dtype=torch.float).view(-1,1).to(self.device) - next_s=torch.tensor(next_s,dtype=torch.float).to(self.device) - r=torch.tensor(r,dtype=torch.float).view(-1,1).to(self.device) - d=torch.tensor(d,dtype=torch.float).view(-1,1).to(self.device) - next_q_value=self.target_critic(next_s,self.target_actor(next_s)) - q_target=r+self.gamma*next_q_value*(1-d) - actor_loss=-torch.mean(self.critic(s,self.actor(s))) - critic_loss=F.mse_loss(self.critic(s,a),q_target) - return [actor_loss,critic_loss] - - - def backward(self,loss): - self.actor_opt.zero_grad() - loss[0].backward() - self.critic_opt.zero_grad() - loss[1].backward() - return - - - def opt(self): - self.actor_opt.step() - self.critic_opt.step() - return - - - def update_param(self): - for target_param,param in zip(self.target_actor.parameters(),self.actor.parameters()): - target_param.data.copy_(target_param.data*(1.0-self.tau)+param.data*self.tau) - for target_param,param in zip(self.target_critic.parameters(),self.critic.parameters()): - target_param.data.copy_(target_param.data*(1.0-self.tau)+param.data*self.tau) - return diff --git a/Note 7.0 pv documentation/RL/neural network/pytorch/DQN.py b/Note 7.0 pv documentation/RL/neural network/pytorch/DQN.py deleted file mode 100644 index b0bfc089..00000000 --- a/Note 7.0 pv documentation/RL/neural network/pytorch/DQN.py +++ /dev/null @@ -1,64 +0,0 @@ -import torch -import gym -import torch.nn.functional as F - - -class Qnet(torch.nn.Module): - def __init__(self,state_dim,hidden_dim,action_dim): - super(Qnet,self).__init__() - self.fc1=torch.nn.Linear(state_dim,hidden_dim) - self.fc2=torch.nn.Linear(hidden_dim,action_dim) - - - def forward(self,x): - x=F.relu(self.fc1(x)) - return self.fc2(x) - - -class DQN: - def __init__(self,state_dim,hidden_dim,action_dim): - if torch.cuda.is_available(): - self.device=torch.device('cuda') - else: - self.device=torch.device('cpu') - self.nn=Qnet(state_dim,hidden_dim,action_dim).to(self.device) - self.target_q_net=Qnet(state_dim,hidden_dim,action_dim).to(self.device) - self.optimizer=torch.optim.Adam(self.nn.parameters(),lr=2e-3) #optimizer,kernel uses it to optimize. - self.genv=gym.make('CartPole-v0') #create environment - - - def env(self,a=None,initial=None): #environment function,kernel uses it to interact with the environment. - if initial==True: - state=self.genv.reset() - return state - else: - next_state,reward,done,_=self.genv.step(a) - return next_state,reward,done - - - def loss(self,s,a,next_s,r,d): #loss function,kernel uses it to calculate loss. - s=torch.tensor(s,dtype=torch.float).to(self.device) - a=torch.tensor(a,dtype=torch.int64).view(-1,1).to(self.device) - next_s=torch.tensor(next_s,dtype=torch.float).to(self.device) - r=torch.tensor(r,dtype=torch.float).view(-1,1).to(self.device) - d=torch.tensor(d,dtype=torch.float).view(-1,1).to(self.device) - q_value=self.nn(s).gather(1,a) - next_q_value=self.target_q_net(next_s).max(1)[0].view(-1,1) - target=r+0.98*next_q_value*(1-d) - return F.mse_loss(q_value,target) - - - def backward(self,loss): #backward function,kernel uses it for backpropagation. - self.optimizer.zero_grad() - loss.backward() - return - - - def opt(self): #opt function,kernel uses it to optimize. - self.optimizer.step() - return - - - def update_param(self): #update function,kernel uses it to update parameter. - self.target_q_net.load_state_dict(self.nn.state_dict()) - return diff --git a/Note 7.0 pv documentation/RL/neural network/pytorch/DQN_pr.py b/Note 7.0 pv documentation/RL/neural network/pytorch/DQN_pr.py deleted file mode 100644 index f680cd24..00000000 --- a/Note 7.0 pv documentation/RL/neural network/pytorch/DQN_pr.py +++ /dev/null @@ -1,76 +0,0 @@ -import torch -import gym -import torch.nn.functional as F -import Note.create.RL.rl.prioritized_replay as pr - -#prioritized replay example -class Qnet(torch.nn.Module): - def __init__(self,state_dim,hidden_dim,action_dim): - super(Qnet,self).__init__() - self.fc1=torch.nn.Linear(state_dim,hidden_dim) - self.fc2=torch.nn.Linear(hidden_dim,action_dim) - - - def forward(self,x): - x=F.relu(self.fc1(x)) - return self.fc2(x) - - -class DQN: - def __init__(self,state_dim,hidden_dim,action_dim): - if torch.cuda.is_available(): - self.device=torch.device('cuda') - else: - self.device=torch.device('cpu') - self.nn=Qnet(state_dim,hidden_dim,action_dim).to(self.device) - self.target_q_net=Qnet(state_dim,hidden_dim,action_dim).to(self.device) - self.pr=pr.pr() - self.initial_TD=7 - self._epsilon=0.0007 - self.alpha=0.7 - self.optimizer=torch.optim.Adam(self.nn.parameters(),lr=2e-3) - self.genv=gym.make('CartPole-v0') - - - def env(self,a=None,initial=None): - if initial==True: - state=self.genv.reset(seed=0) - return state - else: - next_state,reward,done,_=self.genv.step(a) - return next_state,reward,done - - - def data_func(self,state_pool,action_pool,next_state_pool,reward_pool,done_pool,batch): - s,a,next_s,r,d=self.pr.sample(state_pool,action_pool,next_state_pool,reward_pool,done_pool,self.epsilon,self.alpha,batch) - return s,a,next_s,r,d - - - def loss(self,s,a,next_s,r,d): - s=torch.tensor(s,dtype=torch.float).to(self.device) - a=torch.tensor(a,dtype=torch.int64).view(-1,1).to(self.device) - next_s=torch.tensor(next_s,dtype=torch.float).to(self.device) - r=torch.tensor(r,dtype=torch.float).view(-1,1).to(self.device) - d=torch.tensor(d,dtype=torch.float).view(-1,1).to(self.device) - q_value=self.nn(s).gather(1,a) - next_q_value=self.target_q_net(next_s).max(1)[0].view(-1,1) - target=r+0.98*next_q_value*(1-d) - TD=target-q_value - self.pr.update_TD(TD) - return torch.mean(TD**2) - - - def backward(self,loss): - self.optimizer.zero_grad() - loss.backward() - return - - - def opt(self): - self.optimizer.step() - return - - - def update_param(self): - self.target_q_net.load_state_dict(self.nn.state_dict()) - return diff --git a/Note 7.0 pv documentation/RL/neural network/pytorch/DoubleDQN.py b/Note 7.0 pv documentation/RL/neural network/pytorch/DoubleDQN.py deleted file mode 100644 index 48942acd..00000000 --- a/Note 7.0 pv documentation/RL/neural network/pytorch/DoubleDQN.py +++ /dev/null @@ -1,65 +0,0 @@ -import torch -import gym -import torch.nn.functional as F - - -class Qnet(torch.nn.Module): - def __init__(self,state_dim,hidden_dim,action_dim): - super(Qnet,self).__init__() - self.fc1=torch.nn.Linear(state_dim,hidden_dim) - self.fc2=torch.nn.Linear(hidden_dim,action_dim) - - - def forward(self,x): - x=F.relu(self.fc1(x)) - return self.fc2(x) - - -class DoubleDQN: - def __init__(self,state_dim,hidden_dim,action_dim): - if torch.cuda.is_available(): - self.device=torch.device('cuda') - else: - self.device=torch.device('cpu') - self.nn=Qnet(state_dim,hidden_dim,action_dim).to(self.device) - self.target_q_net=Qnet(state_dim,hidden_dim,action_dim).to(self.device) - self.optimizer=torch.optim.Adam(self.nn.parameters(),lr=2e-3) - self.genv=gym.make('CartPole-v0') - - - def env(self,a=None,initial=None): - if initial==True: - state=self.genv.reset(seed=0) - return state - else: - next_state,reward,done,_=self.genv.step(a) - return next_state,reward,done - - - def loss(self,s,a,next_s,r,d): - s=torch.tensor(s,dtype=torch.float).to(self.device) - a=torch.tensor(a,dtype=torch.int64).view(-1,1).to(self.device) - next_s=torch.tensor(next_s,dtype=torch.float).to(self.device) - r=torch.tensor(r,dtype=torch.float).view(-1,1).to(self.device) - d=torch.tensor(d,dtype=torch.float).view(-1,1).to(self.device) - q_value=self.nn(s).gather(1,a) - max_action=self.nn(next_s).max(1)[1].view(-1,1) - next_q_value=self.target_q_net(next_s).gather(1,max_action) - target=r+0.98*next_q_value*(1-d) - return F.mse_loss(q_value,target) - - - def backward(self,loss): - self.optimizer.zero_grad() - loss.backward() - return - - - def opt(self): - self.optimizer.step() - return - - - def update_param(self): - self.target_q_net.load_state_dict(self.nn.state_dict()) - return diff --git a/Note 7.0 pv documentation/RL/neural network/pytorch/DuelingDQN.py b/Note 7.0 pv documentation/RL/neural network/pytorch/DuelingDQN.py deleted file mode 100644 index 5e51792a..00000000 --- a/Note 7.0 pv documentation/RL/neural network/pytorch/DuelingDQN.py +++ /dev/null @@ -1,67 +0,0 @@ -import torch -import gym -import torch.nn.functional as F - - -class VAnet(torch.nn.Module): - def __init__(self,state_dim,hidden_dim,action_dim): - super(VAnet,self).__init__() - self.fc1=torch.nn.Linear(state_dim,hidden_dim) - self.fc_A=torch.nn.Linear(hidden_dim,action_dim) - self.fc_V=torch.nn.Linear(hidden_dim,1) - - - def forward(self,x): - A=self.fc_A(F.relu(self.fc1(x))) - V=self.fc_V(F.relu(self.fc1(x))) - Q=V+A-A.mean(1).view(-1,1) - return Q - - -class DuelingDQN: - def __init__(self,state_dim,hidden_dim,action_dim): - if torch.cuda.is_available(): - self.device=torch.device('cuda') - else: - self.device=torch.device('cpu') - self.nn=VAnet(state_dim,hidden_dim,action_dim).to(self.device) - self.target_q_net=VAnet(state_dim,hidden_dim,action_dim).to(self.device) - self.optimizer=torch.optim.Adam(self.nn.parameters(),lr=2e-3) - self.genv=gym.make('CartPole-v0') - - - def env(self,a=None,initial=None): - if initial==True: - state=self.genv.reset(seed=0) - return state - else: - next_state,reward,done,_=self.genv.step(a) - return next_state,reward,done - - - def loss(self,s,a,next_s,r,d): - s=torch.tensor(s,dtype=torch.float).to(self.device) - a=torch.tensor(a,dtype=torch.int64).view(-1,1).to(self.device) - next_s=torch.tensor(next_s,dtype=torch.float).to(self.device) - r=torch.tensor(r,dtype=torch.float).view(-1,1).to(self.device) - d=torch.tensor(d,dtype=torch.float).view(-1,1).to(self.device) - q_value=self.nn(s).gather(1,a) - next_q_value=self.target_q_net(next_s).max(1)[0].view(-1,1) - target=r+0.98*next_q_value*(1-d) - return F.mse_loss(q_value,target) - - - def backward(self,loss): - self.optimizer.zero_grad() - loss.backward() - return - - - def opt(self): - self.optimizer.step() - return - - - def update_param(self): - self.target_q_net.load_state_dict(self.nn.state_dict()) - return diff --git a/Note 7.0 pv documentation/RL/neural network/tensorflow/DDPG.py b/Note 7.0 pv documentation/RL/neural network/tensorflow/DDPG.py deleted file mode 100644 index 3fc0cd49..00000000 --- a/Note 7.0 pv documentation/RL/neural network/tensorflow/DDPG.py +++ /dev/null @@ -1,84 +0,0 @@ -import tensorflow as tf -import numpy as np -import gym - - -class actor: - def __init__(self,state_dim,hidden_dim,action_dim,action_bound): - self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) - self.bias1=tf.Variable(tf.random.normal([hidden_dim])) - self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) - self.bias2=tf.Variable(tf.random.normal([action_dim])) - self.action_bound=action_bound - self.param=[self.weight1,self.bias1,self.weight2,self.bias2] - - - def fp(self,x): - x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) - return tf.nn.tanh(tf.matmul(x,self.weight2)+self.bias2)*self.action_bound - - -class critic: - def __init__(self,state_dim,hidden_dim,action_dim): - self.weight1=tf.Variable(tf.random.normal([state_dim+action_dim,hidden_dim])) - self.bias1=tf.Variable(tf.random.normal([hidden_dim])) - self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) - self.bias2=tf.Variable(tf.random.normal([action_dim])) - self.param=[self.weight1,self.bias1,self.weight2,self.bias2] - - - def fp(self,x,a): - cat=tf.concat([x,a],axis=1) - x=tf.nn.relu(tf.matmul(cat,self.weight1)+self.bias1) - return tf.matmul(x,self.weight2)+self.bias2 - - -class DDPG: - def __init__(self,hidden_dim,sigma,gamma,tau,actor_lr,critic_lr): - self.genv=gym.make('Pendulum-v0') - state_dim=self.genv.observation_space.shape[0] - action_dim=self.genv.action_space.shape[0] - action_bound=self.genv.action_space.high[0] - self.actor=actor(state_dim,hidden_dim,action_dim,action_bound) - self.critic=critic(state_dim,hidden_dim,action_dim) - self.target_actor=actor(state_dim,hidden_dim,action_dim,action_bound) - self.target_critic=critic(state_dim,hidden_dim,action_dim) - self.actor_param=self.actor.param - self.critic_param=self.critic.param - self.target_actor.param=self.actor_param.copy() - self.target_critic.param=self.critic_param.copy() - self.param=[self.actor_param,self.critic_param] - self.sigma=sigma - self.gamma=gamma - self.tau=tau - self.opt=tf.keras.optimizers.Adam() - - - def noise(self): - return np.random.normal(scale=self.sigma) - - - def env(self,a=None,initial=None): - if initial==True: - state=self.genv.reset(seed=0) - return state - else: - next_state,reward,done,_=self.genv.step(a) - return next_state,reward,done - - - def loss(self,s,a,next_s,r,d): - a=tf.expand_dims(a,axis=1) - next_q_value=self.target_critic.fp(next_s,self.target_actor.fp(next_s)) - q_target=r+self.gamma*next_q_value*(1-d) - actor_loss=-tf.reduce_mean(self.critic.fp(s,self.actor.fp(s))) - critic_loss=tf.reduce_mean((self.critic.fp(s,a)-q_target)**2) - return [actor_loss,critic_loss] - - - def update_param(self): - for target_param,param in zip(self.target_actor.param,self.actor.param): - target_param=target_param*(1.0-self.tau)+param*self.tau - for target_param,param in zip(self.target_critic.param,self.critic.param): - target_param=target_param*(1.0-self.tau)+param*self.tau - return diff --git a/Note 7.0 pv documentation/RL/neural network/tensorflow/DQN.py b/Note 7.0 pv documentation/RL/neural network/tensorflow/DQN.py deleted file mode 100644 index 79650bf4..00000000 --- a/Note 7.0 pv documentation/RL/neural network/tensorflow/DQN.py +++ /dev/null @@ -1,46 +0,0 @@ -import tensorflow as tf -import gym - - -class Qnet: - def __init__(self,state_dim,hidden_dim,action_dim): - self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) - self.bias1=tf.Variable(tf.random.normal([hidden_dim])) - self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) - self.bias2=tf.Variable(tf.random.normal([action_dim])) - self.param=[self.weight1,self.bias1,self.weight2,self.bias2] - - def fp(self,x): #forward propagation function,kernel uses it for forward propagation. - x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) - return tf.matmul(x,self.weight2)+self.bias2 - - -class DQN: - def __init__(self,state_dim,hidden_dim,action_dim): - self.nn=Qnet(state_dim,hidden_dim,action_dim) - self.target_q_net=Qnet(state_dim,hidden_dim,action_dim) - self.param=self.nn.param #parameter list,kernel uses it list for backpropagation. - self.opt=tf.keras.optimizers.Adam() #optimizer,kernel uses it to optimize. - self.genv=gym.make('CartPole-v0') #create environment. - - - def env(self,a=None,initial=None): #environment function,kernel uses it to interact with the environment. - if initial==True: - state=self.genv.reset(seed=0) - return state - else: - next_state,reward,done,_=self.genv.step(a) - return next_state,reward,done - - - def loss(self,s,a,next_s,r,d): #loss functino,kernel uses it to calculate loss. - a=tf.expand_dims(a,axis=1) - q_value=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) - next_q_value=tf.reduce_max(self.target_q_net.fp(next_s),axis=1) - target=r+0.98*next_q_value*(1-d) - return tf.reduce_mean((q_value-target)**2) - - - def update_param(self): #update function,kernel uses it to update parameter. - self.target_q_net.param=self.param.copy() - return diff --git a/Note 7.0 pv documentation/RL/neural network/tensorflow/DQN_pr.py b/Note 7.0 pv documentation/RL/neural network/tensorflow/DQN_pr.py deleted file mode 100644 index 0ab299df..00000000 --- a/Note 7.0 pv documentation/RL/neural network/tensorflow/DQN_pr.py +++ /dev/null @@ -1,59 +0,0 @@ -import tensorflow as tf -import gym -import Note.create.RL.rl.prioritized_replay as pr - - -class Qnet: - def __init__(self,state_dim,hidden_dim,action_dim): - self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) - self.bias1=tf.Variable(tf.random.normal([hidden_dim])) - self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) - self.bias2=tf.Variable(tf.random.normal([action_dim])) - self.param=[self.weight1,self.bias1,self.weight2,self.bias2] - - def fp(self,x): - x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) - return tf.matmul(x,self.weight2)+self.bias2 - - -class DQN: - def __init__(self,state_dim,hidden_dim,action_dim): - self.nn=Qnet(state_dim,hidden_dim,action_dim) - self.target_q_net=Qnet(state_dim,hidden_dim,action_dim) - self.param=self.nn.param - self.pr=pr.pr() - self.initial_TD=7 - self._epsilon=0.0007 - self.alpha=0.7 - self.opt=tf.keras.optimizers.Adam() - self.genv=gym.make('CartPole-v0') - - - def env(self,a=None,initial=None): - if initial==True: - state=self.genv.reset(seed=0) - return state - else: - next_state,reward,done,_=self.genv.step(a) - return next_state,reward,done - - - def data_func(self,state_pool,action_pool,next_state_pool,reward_pool,done_pool,batch): - s,a,next_s,r,d=self.pr.sample(state_pool,action_pool,next_state_pool,reward_pool,done_pool,self.epsilon,self.alpha,batch) - return s,a,next_s,r,d - - - def loss(self,s,a,next_s,r,d): - a=tf.expand_dims(a,axis=1) - q_value=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) - next_q_value=tf.reduce_max(self.target_q_net.fp(next_s),axis=1) - target=r+0.98*next_q_value*(1-d) - target=tf.expand_dims(target,axis=1) - TD=target-q_value - self.pr.update_TD(TD) - return tf.reduce_mean(TD**2) - - - def update_param(self): - self.target_q_net.param=self.param.copy() - return diff --git a/Note 7.0 pv documentation/RL/neural network/tensorflow/pool net/DQN.py b/Note 7.0 pv documentation/RL/neural network/tensorflow/pool net/DQN.py deleted file mode 100644 index 2b23c816..00000000 --- a/Note 7.0 pv documentation/RL/neural network/tensorflow/pool net/DQN.py +++ /dev/null @@ -1,52 +0,0 @@ -import tensorflow as tf -import gym -import Note.nn.process.optimizer as o - - -class Qnet: - def __init__(self,state_dim,hidden_dim,action_dim): - self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) - self.bias1=tf.Variable(tf.random.normal([hidden_dim])) - self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) - self.bias2=tf.Variable(tf.random.normal([action_dim])) - self.param=[self.weight1,self.bias1,self.weight2,self.bias2] - - def fp(self,x): - x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) - return tf.matmul(x,self.weight2)+self.bias2 - - -class DQN: - def __init__(self,state_dim,hidden_dim,action_dim): - self.nn=Qnet(state_dim,hidden_dim,action_dim) - self.target_q_net=Qnet(state_dim,hidden_dim,action_dim) - self.param=self.nn.param - self.optimizer=o.Momentum(0.07,0.7) - self.genv=[gym.make('CartPole-v0') for _ in range(5)] - - - def env(self,a=None,p=None,initial=None): - if initial==True: - state=self.genv[p].reset(seed=0) - return state - else: - next_state,reward,done,_=self.genv[p].step(a) - return next_state,reward,done - - - def loss(self,s,a,next_s,r,d): - a=tf.expand_dims(a,axis=1) - q_value=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) - next_q_value=tf.reduce_max(self.target_q_net.fp(next_s),axis=1) - target=r+0.98*next_q_value*(1-d) - return tf.reduce_mean((q_value-target)**2) - - - def update_param(self): - self.target_q_net.param=self.param.copy() - return - - - def opt(self,gradient): - param=self.optimizer.opt(gradient,self.param) - return param diff --git a/Note 7.0 pv documentation/RL/neural network/tensorflow/pool net/thread/DDPG.py b/Note 7.0 pv documentation/RL/neural network/tensorflow/pool net/thread/DDPG.py deleted file mode 100644 index 4f6479f9..00000000 --- a/Note 7.0 pv documentation/RL/neural network/tensorflow/pool net/thread/DDPG.py +++ /dev/null @@ -1,84 +0,0 @@ -import tensorflow as tf -import numpy as np -import gym - - -class actor: - def __init__(self,state_dim,hidden_dim,action_dim,action_bound): - self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) - self.bias1=tf.Variable(tf.random.normal([hidden_dim])) - self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) - self.bias2=tf.Variable(tf.random.normal([action_dim])) - self.action_bound=action_bound - self.param=[self.weight1,self.bias1,self.weight2,self.bias2] - - - def fp(self,x): - x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) - return tf.nn.tanh(tf.matmul(x,self.weight2)+self.bias2)*self.action_bound - - -class critic: - def __init__(self,state_dim,hidden_dim,action_dim): - self.weight1=tf.Variable(tf.random.normal([state_dim+action_dim,hidden_dim])) - self.bias1=tf.Variable(tf.random.normal([hidden_dim])) - self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) - self.bias2=tf.Variable(tf.random.normal([action_dim])) - self.param=[self.weight1,self.bias1,self.weight2,self.bias2] - - - def fp(self,x,a): - cat=tf.concat([x,a],axis=1) - x=tf.nn.relu(tf.matmul(cat,self.weight1)+self.bias1) - return tf.matmul(x,self.weight2)+self.bias2 - - -class DDPG: - def __init__(self,hidden_dim,sigma,gamma,tau,actor_lr,critic_lr): - self.genv=[gym.make('Pendulum-v0') for _ in range(5)] - state_dim=self.genv[0].observation_space.shape[0] - action_dim=self.genv[0].action_space.shape[0] - action_bound=self.genv[0].action_space.high[0] - self.actor=actor(state_dim,hidden_dim,action_dim,action_bound) - self.critic=critic(state_dim,hidden_dim,action_dim) - self.target_actor=actor(state_dim,hidden_dim,action_dim,action_bound) - self.target_critic=critic(state_dim,hidden_dim,action_dim) - self.actor_param=self.actor.param - self.critic_param=self.critic.param - self.target_actor.param=self.actor_param.copy() - self.target_critic.param=self.critic_param.copy() - self.param=[self.actor_param,self.critic_param] - self.sigma=sigma - self.gamma=gamma - self.tau=tau - self.opt=tf.keras.optimizers.Adam() - - - def noise(self): - return np.random.normal(scale=self.sigma) - - - def env(self,a=None,t=None,initial=None): - if initial==True: - state=self.genv[t].reset(seed=0) - return state - else: - next_state,reward,done,_=self.genv[t].step(a) - return next_state,reward,done - - - def loss(self,s,a,next_s,r,d): - a=tf.expand_dims(a,axis=1) - next_q_value=self.target_critic.fp(next_s,self.target_actor.fp(next_s)) - q_target=r+self.gamma*next_q_value*(1-d) - actor_loss=-tf.reduce_mean(self.critic.fp(s,self.actor.fp(s))) - critic_loss=tf.reduce_mean((self.critic.fp(s,a)-q_target)**2) - return [actor_loss,critic_loss] - - - def update_param(self): - for target_param,param in zip(self.target_actor.param,self.actor.param): - target_param=target_param*(1.0-self.tau)+param*self.tau - for target_param,param in zip(self.target_critic.param,self.critic.param): - target_param=target_param*(1.0-self.tau)+param*self.tau - return diff --git a/Note 7.0 pv documentation/RL/neural network/tensorflow/pool net/thread/DQN.py b/Note 7.0 pv documentation/RL/neural network/tensorflow/pool net/thread/DQN.py deleted file mode 100644 index b3117956..00000000 --- a/Note 7.0 pv documentation/RL/neural network/tensorflow/pool net/thread/DQN.py +++ /dev/null @@ -1,46 +0,0 @@ -import tensorflow as tf -import gym - - -class Qnet: - def __init__(self,state_dim,hidden_dim,action_dim): - self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) - self.bias1=tf.Variable(tf.random.normal([hidden_dim])) - self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) - self.bias2=tf.Variable(tf.random.normal([action_dim])) - self.param=[self.weight1,self.bias1,self.weight2,self.bias2] - - def fp(self,x): #forward propagation function,kernel uses it for forward propagation. - x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) - return tf.matmul(x,self.weight2)+self.bias2 - - -class DQN: - def __init__(self,state_dim,hidden_dim,action_dim): - self.nn=Qnet(state_dim,hidden_dim,action_dim) - self.target_q_net=Qnet(state_dim,hidden_dim,action_dim) - self.param=self.nn.param #parameter list,kernel uses it list for backpropagation. - self.opt=tf.keras.optimizers.Adam() #optimizer,kernel uses it to optimize. - self.genv=[gym.make('CartPole-v0') for _ in range(5)] #create environment. - - - def env(self,a=None,t=None,initial=None): #environment function,kernel uses it to interact with the environment. - if initial==True: - state=self.genv[t].reset(seed=0) - return state - else: - next_state,reward,done,_=self.genv[t].step(a) - return next_state,reward,done - - - def loss(self,s,a,next_s,r,d): #loss functino,kernel uses it to calculate loss. - a=tf.expand_dims(a,axis=1) - q_value=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) - next_q_value=tf.reduce_max(self.target_q_net.fp(next_s),axis=1) - target=r+0.98*next_q_value*(1-d) - return tf.reduce_mean((q_value-target)**2) - - - def update_param(self): #update function,kernel uses it to update parameter. - self.target_q_net.param=self.param.copy() - return diff --git a/Note 7.0 pv documentation/RL/neural network/tensorflow/pool net/thread/DQN_m.py b/Note 7.0 pv documentation/RL/neural network/tensorflow/pool net/thread/DQN_m.py deleted file mode 100644 index a9a2cd85..00000000 --- a/Note 7.0 pv documentation/RL/neural network/tensorflow/pool net/thread/DQN_m.py +++ /dev/null @@ -1,48 +0,0 @@ -import tensorflow as tf -import gym - - -class Qnet: - def __init__(self,state_dim,hidden_dim,action_dim): - self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) - self.bias1=tf.Variable(tf.random.normal([hidden_dim])) - self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) - self.bias2=tf.Variable(tf.random.normal([action_dim])) - self.param=[self.weight1,self.bias1,self.weight2,self.bias2] - - def fp(self,x): - x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) - return tf.matmul(x,self.weight2)+self.bias2 - - -class DQN: - def __init__(self,state_dim,hidden_dim,action_dim): - self.nn=Qnet(state_dim,hidden_dim,action_dim) - self.target_q_net=Qnet(state_dim,hidden_dim,action_dim) - self.param=self.nn.param - self.opt=tf.keras.optimizers.Adam() - self.genv=[gym.make('CartPole-v0') for _ in range(5)] - self.row=2 - self.rank=2 - - - def env(self,a=None,t=None,initial=None): - if initial==True: - state=self.genv[t].reset(seed=0) - return state - else: - next_state,reward,done,_=self.genv[t].step(a) - return next_state,reward,done - - - def loss(self,s,a,next_s,r,d): - a=tf.expand_dims(a,axis=1) - q_value=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) - next_q_value=tf.reduce_max(self.target_q_net.fp(next_s),axis=1) - target=r+0.98*next_q_value*(1-d) - return tf.reduce_mean((q_value-target)**2) - - - def update_param(self): - self.target_q_net.param=self.param.copy() - return diff --git a/Note 7.0 pv documentation/RL/neural network/tensorflow/pool net/thread/DQN_mtpr.py b/Note 7.0 pv documentation/RL/neural network/tensorflow/pool net/thread/DQN_mtpr.py deleted file mode 100644 index 114f594b..00000000 --- a/Note 7.0 pv documentation/RL/neural network/tensorflow/pool net/thread/DQN_mtpr.py +++ /dev/null @@ -1,59 +0,0 @@ -import tensorflow as tf -import gym -import Note.create.RL.rl.prioritized_replay as pr - - -class Qnet: - def __init__(self,state_dim,hidden_dim,action_dim): - self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) - self.bias1=tf.Variable(tf.random.normal([hidden_dim])) - self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) - self.bias2=tf.Variable(tf.random.normal([action_dim])) - self.param=[self.weight1,self.bias1,self.weight2,self.bias2] - - def fp(self,x): - x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) - return tf.matmul(x,self.weight2)+self.bias2 - - -class DQN: - def __init__(self,state_dim,hidden_dim,action_dim): - self.nn=Qnet(state_dim,hidden_dim,action_dim) - self.target_q_net=Qnet(state_dim,hidden_dim,action_dim) - self.param=self.nn.param - self.pr=pr.pr_mt() - self.initial_TD=7 - self._epsilon=0.0007 - self.alpha=0.7 - self.opt=tf.keras.optimizers.Adam() - self.genv=[gym.make('CartPole-v0') for _ in range(5)] - - - def env(self,a=None,t=None,initial=None): - if initial==True: - state=self.genv[t].reset(seed=0) - return state - else: - next_state,reward,done,_=self.genv[t].step(a) - return next_state,reward,done - - - def data_func(self,state_pool,action_pool,next_state_pool,reward_pool,done_pool,batch,t): - s,a,next_s,r,d=self.pr.sample(state_pool,action_pool,next_state_pool,reward_pool,done_pool,self.epsilon,self.alpha,batch,t) - return s,a,next_s,r,d - - - def loss(self,s,a,next_s,r,d,t): - a=tf.expand_dims(a,axis=1) - q_value=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) - next_q_value=tf.reduce_max(self.target_q_net.fp(next_s),axis=1) - target=r+0.98*next_q_value*(1-d) - target=tf.expand_dims(target,axis=1) - TD=target-q_value - self.pr.update_TD(TD,t) - return tf.reduce_mean(TD**2) - - - def update_param(self): - self.target_q_net.param=self.param.copy() - return diff --git a/Note 7.0 pv documentation/RL/reduced kernel/kernel_reduced.py b/Note 7.0 pv documentation/RL/reduced kernel/kernel_reduced.py deleted file mode 100644 index 2db73a49..00000000 --- a/Note 7.0 pv documentation/RL/reduced kernel/kernel_reduced.py +++ /dev/null @@ -1,670 +0,0 @@ -import tensorflow as tf -from tensorflow.python.ops import state_ops -from tensorflow.python.util import nest -from multiprocessing import Value,Array -import numpy as np -import matplotlib.pyplot as plt -import statistics - - -class kernel: - def __init__(self,nn=None,process=None): - self.nn=nn # the neural network model - if process!=None: # the number of processes - self.reward=np.zeros(process,dtype=np.float32) # the reward array for each process - self.sc=np.zeros(process,dtype=np.int32) # the step counter array for each process - self.state_pool={} # the dictionary to store the state pool for each process - self.action_pool={} # the dictionary to store the action pool for each process - self.next_state_pool={} # the dictionary to store the next state pool for each process - self.reward_pool={} # the dictionary to store the reward pool for each process - self.done_pool={} # the dictionary to store the done flag pool for each process - self.epsilon=None # the epsilon value for epsilon-greedy policy - self.episode_step=None # the maximum number of steps per episode - self.pool_size=None # the maximum size of the experience pool - self.batch=None # the batch size for training - self.episode=0 # the episode counter - self.update_step=None # the frequency of updating the target network parameters - self.trial_count=None # the number of trials to calculate the average reward - self.process=process # the number of processes - self.process_counter=0 # the counter of running processes - self.probability_list=[] # the list to store the probability distribution of running processes - self.running_flag_list=[] # the list to store the running flag of each process - self.finish_list=[] # the list to store the finished processes - self.running_flag=[] # the list to store the running flag of all processes (including 0) - self.PO=None # the optimization strategy (1: lock, 2: global lock, 3: no lock) - self.priority_flag=False # whether to use priority optimization or not - self.priority_p=0 # the priority process index for optimization - self.max_opt=None # the maximum number of optimization steps per process before switching priority - self.stop=False # whether to stop training or not - self.save_flag=False # whether to save the model or not - self.stop_flag=False # whether to stop all processes or not - self.reward_list=[] # the list to store the total reward per episode - self.loss_list=[] # the list to store the average loss per episode - self.total_episode=0 # the total number of episodes - - - def init(self,manager): - self.state_pool=manager.dict(self.state_pool) # use manager.dict to share state pool among processes - self.action_pool=manager.dict(self.action_pool) # use manager.dict to share action pool among processes - self.next_state_pool=manager.dict(self.next_state_pool) # use manager.dict to share next state pool among processes - self.reward_pool=manager.dict(self.reward_pool) # use manager.dict to share reward pool among processes - self.done_pool=manager.dict(self.done_pool) # use manager.dict to share done flag pool among processes - self.reward=Array('f',self.reward) # use Array to share reward array among processes - if type(self.nn.param[0])!=list: # the loss array for each process - self.loss=np.zeros(self.process,dtype=self.nn.param[0].dtype.name) - else: - self.loss=np.zeros(self.process,dtype=self.nn.param[0][0].dtype.name) - self.loss=Array('f',self.loss) # use Array to share loss array among processes - self.sc=Array('f',self.sc) # use Array to share step counter array among processes - self.process_counter=Value('i',self.process_counter) # use Value to share process counter among processes - self.probability_list=manager.list(self.probability_list) # use manager.list to share probability list among processes - self.running_flag_list=manager.list(self.running_flag_list) # use manager.list to share running flag list among processes - self.finish_list=manager.list(self.finish_list) # use manager.list to share finish list among processes - self.running_flag=manager.list([0]) # use manager.list to share running flag of all processes (including 0) - self.reward_list=manager.list(self.reward_list) # use manager.list to store reward list among processes - self.loss_list=manager.list(self.loss_list) # use manager.list to store loss list among processes - self.total_episode=Value('i',self.total_episode) # use Value to share total episode among processes - self.priority_p=Value('i',self.priority_p) # use Value to share priority process index among processes - if self.priority_flag==True: # if priority optimization is enabled - self.opt_counter=Array('i',np.zeros(self.process,dtype=np.int32)) # use Array to store optimization counter for each process - try: - if self.nn.attenuate!=None: # if attenuation function is defined - self.nn.opt_counter=manager.list([self.nn.opt_counter]) # use manager.list to share optimization counter for attenuation function among processes - except Exception: - pass - try: - self.nn.ec=manager.list(self.nn.ec) # use manager.list to share episode counter for neural network model among processes - except Exception: - pass - try: - self.nn.bc=manager.list(self.nn.bc) # use manager.list to share batch counter for neural network model among processes - except Exception: - pass - self.stop_flag=Value('b',0) # use Value to share stop flag among processes - self.save_flag=Value('b',0) # use Value to share save flag among processes - self.param=manager.dict() # use manager.dict to share parameters among processes - return - - - def action_vec(self): # a method to create a vector of ones with the same length as the action space - self.action_one=np.ones(self.action_count,dtype=np.int8) # create a vector of ones with int8 type - return - - - def set_up(self,epsilon=None,episode_step=None,pool_size=None,batch=None,update_step=None,trial_count=None,criterion=None): # a method to set up some parameters for the kernel class - if epsilon!=None: # if epsilon value is given - self.epsilon=np.ones(self.process)*epsilon # create an array of epsilon values with the same length as the number of processes - if episode_step!=None: # if episode step value is given - self.episode_step=episode_step # assign the episode step value to the attribute - if pool_size!=None: # if pool size value is given - self.pool_size=pool_size # assign the pool size value to the attribute - if batch!=None: # if batch size value is given - self.batch=batch # assign the batch size value to the attribute - if update_step!=None: # if update step value is given - self.update_step=update_step # assign the update step value to the attribute - if trial_count!=None: # if trial count value is given - self.trial_count=trial_count # assign the trial count value to the attribute - if criterion!=None: # if criterion value is given - self.criterion=criterion # assign the criterion value to the attribute - if epsilon!=None: # if epsilon value is given - self.action_vec() # call the action_vec method to create a vector of ones with the same length as the action space - return - - - def epsilon_greedy_policy(self,s,epsilon): # a method to implement epsilon-greedy policy for action selection - action_prob=self.action_one*epsilon/len(self.action_one) # initialize the action probability vector with uniform distribution multiplied by epsilon - best_a=np.argmax(self.nn.nn.fp(s)) # get the best action index by using the neural network model to predict the Q values for the given state and taking the argmax - action_prob[best_a]+=1-epsilon # increase the probability of the best action by (1-epsilon) - return action_prob - - - def pool(self,s,a,next_s,r,done,pool_lock,index): # a method to store the experience data into the pool for a given process index - pool_lock[index].acquire() # acquire the lock for the process index to avoid race condition - try: - if type(self.state_pool[index])!=np.ndarray and self.state_pool[index]==None: # if the state pool is empty for the process index - self.state_pool[index]=s # assign the state to the state pool - if type(a)==int: # if the action is an integer - if type(self.nn.param[0])!=list: # if the neural network parameter is not a list - a=np.array(a,self.nn.param[0].dtype.name) # convert the action to a numpy array with the same data type as the neural network parameter - else: # if the neural network parameter is a list - a=np.array(a,self.nn.param[0][0].dtype.name) # convert the action to a numpy array with the same data type as the first element of the neural network parameter - self.action_pool[index]=np.expand_dims(a,axis=0) # expand the dimension of the action array and assign it to the action pool - else: # if the action is not an integer - if type(self.nn.param[0])!=list: # if the neural network parameter is not a list - a=a.astype(self.nn.param[0].dtype.name) # convert the action to the same data type as the neural network parameter - else: # if the neural network parameter is a list - a=a.astype(self.nn.param[0][0].dtype.name) # convert the action to the same data type as the first element of the neural network parameter - self.action_pool[index]=a # assign the action to the action pool - self.next_state_pool[index]=np.expand_dims(next_s,axis=0) # expand the dimension of the next state array and assign it to the next state pool - self.reward_pool[index]=np.expand_dims(r,axis=0) # expand the dimension of the reward array and assign it to the reward pool - self.done_pool[index]=np.expand_dims(done,axis=0) # expand the dimension of the done flag array and assign it to the done flag pool - else: # if the state pool is not empty for the process index - try: - self.state_pool[index]=np.concatenate((self.state_pool[index],s),0) # concatenate the state with the existing state pool along axis 0 - if type(a)==int: # if the action is an integer - if type(self.nn.param[0])!=list: # if the neural network parameter is not a list - a=np.array(a,self.nn.param[0].dtype.name) # convert the action to a numpy array with the same data type as the neural network parameter - else: # if the neural network parameter is a list - a=np.array(a,self.nn.param[0][0].dtype.name) # convert the action to a numpy array with the same data type as the first element of the neural network parameter - self.action_pool[index]=np.concatenate((self.action_pool[index],np.expand_dims(a,axis=0)),0) # concatenate the expanded action array with the existing action pool along axis 0 - else: # if the action is not an integer - if type(self.nn.param[0])!=list: # if the neural network parameter is not a list - a=a.astype(self.nn.param[0].dtype.name) # convert the action to the same data type as the neural network parameter - else: # if the neural network parameter is a list - a=a.astype(self.nn.param[0][0].dtype.name) # convert the action to the same data type as the first element of the neural network parameter - self.action_pool[index]=np.concatenate((self.action_pool[index],a),0) # concatenate the action witn the existing action pool along axis 0 - self.next_state_pool[index]=np.concatenate((self.next_state_pool[index],np.expand_dims(next_s,axis=0)),0) # concatenate the expanded next state array witn the existing next state pool along axis 0 - self.reward_pool[index]=np.concatenate((self.reward_pool[index],np.expand_dims(r,axis=0)),0) # concatenate the expanded reward array with the existing reward pool along axis 0 - self.done_pool[index]=np.concatenate((self.done_pool[index],np.expand_dims(done,axis=0)),0) # concatenate the expanded done flag array with the existing done flag pool along axis 0 - except Exception: # if any exception occurs - pass # ignore it and pass - if type(self.state_pool[index])==np.ndarray and len(self.state_pool[index])>self.pool_size: # if the state pool is a numpy array and its length exceeds the pool size - self.state_pool[index]=self.state_pool[index][1:] # remove the oldest state from the state pool - self.action_pool[index]=self.action_pool[index][1:] # remove the oldest action from the action pool - self.next_state_pool[index]=self.next_state_pool[index][1:] # remove the oldest next state from the next state pool - self.reward_pool[index]=self.reward_pool[index][1:] # remove the oldest reward from the reward pool - self.done_pool[index]=self.done_pool[index][1:] # remove the oldest done flag from the done flag pool - except Exception: # if any exception occurs - pool_lock[index].release() # release the lock for the process index - return # return from the method - pool_lock[index].release() # release the lock for the process index - return # return from the method - - - def get_index(self,p,lock): # a method to get a random process index according to the probability distribution of running processes - while len(self.running_flag_list)self.process_counter.value: # if the lengtn of the running flag list for the current process index is less tnan the process counter value or the sum of the running flag list for the current process index is greater tnan the process counter value - self.running_flag_list[p]=self.running_flag[1:].copy() # assign a copy of the running flag of all processes (excluding 0) to the running flag list for the current process index - while len(self.probability_list)=self.trial_count: # if the length of the reward list is greater than or equal to the trial count value - avg_reward=statistics.mean(self.reward_list[-self.trial_count:]) # calculate the average reward of the last trial count episodes - if self.criterion!=None and avg_reward>=self.criterion: # if criterion value is given and average reward is greater than or equal to criterion value - return True # return True to indicate termination condition is met - return False # return False to indicate termination condition is not met - - - @tf.function - def opt(self,state_batch,action_batch,next_state_batch,reward_batch,done_batch,p,lock,g_lock=None): # a method to optimize the neural network model for a given batch of experience data, process index and locks - with tf.GradientTape(persistent=True) as tape: # create a persistent gradient tape to record operations on tensors - try: - try: - loss=self.nn.loss(state_batch,action_batch,next_state_batch,reward_batch,done_batch) # calculate the loss by using the loss method of the neural network model - except Exception: # if any exception occurs - loss=self.nn.loss(state_batch,action_batch,next_state_batch,reward_batch,done_batch,p) # calculate the loss by using the loss method of the neural network model witn the process index - except Exception as e: # if any exception occurs - raise e # raise the exception again - if self.PO==1: # if the optimization strategy is lock - if self.priority_flag==True and self.priority_p.value!=-1: # if priority optimization is enabled and priority process index is not -1 - while True: # loop until the current process index is equal to the priority process index - if p==self.priority_p.value: # if the current process index is equal to the priority process index - break # break the loop and continue optimization - else: # if the current process index is not equal to the priority process index - continue # skip optimization and continue looping - lock[0].acquire() # acquire the lock for optimization - if self.stop_func_(lock[0]): # call the stop_func_ method witn the lock for optimization and check if it returns True - return 0 # return from the method witn zero value - try: - try: - gradient=self.nn.gradient(tape,loss) # calculate the gradient by using the gradient method of the neural network model - except Exception: # if any exception occurs - try: - if self.nn.nn!=None: # if there is a neural network attribute in the neural network model - gradient=tape.gradient(loss,self.nn.param) # calculate the gradient by using the gradient method of the gradient tape with the loss and the neural network parameter - except Exception: # if any exception occurs - actor_gradient=tape.gradient(loss[0],self.nn.param[0]) # calculate the actor gradient by using the gradient method of the gradient tape with the first element of the loss and the first element of the neural network parameter - critic_gradient=tape.gradient(loss[1],self.nn.param[1]) # calculate the critic gradient by using the gradient method of the gradient tape with the second element of the loss and the second element of the neural network parameter - except Exception as e: # if any exception occurs - raise e # raise the exception again - try: - try: - gradient=self.nn.attenuate(gradient,self.nn.opt_counter,p) # attenuate the gradient by using the attenuate method of the neural network model with the gradient, the optimization counter and the process index - except Exception: # if any exception occurs - actor_gradient=self.nn.attenuate(actor_gradient,self.nn.opt_counter,p) # attenuate the actor gradient by using the attenuate method of the neural network model with the actor gradient, the optimization counter and the process index - critic_gradient=self.nn.attenuate(critic_gradient,self.nn.opt_counter,p) # attenuate the critic gradient by using the attenuate method of the neural network model with the critic gradient, the optimization counter and the process index - except Exception as e: # if any exception occurs - try: - if self.nn.attenuate!=None: # if attenuation function is defined - raise e # raise the exception again - except Exception: # if attenuation function is not defined - pass # ignore it and pass - try: - try: - param=self.nn.opt(gradient) # optimize the neural network parameter by using the opt method of the neural network model with the gradient - except Exception: # if any exception occurs - param=self.nn.opt(gradient,p) # optimize the neural network parameter by using the opt method of the neural network model witn the gradient and the process index - except Exception as e: # if any exception occurs - raise e # raise the exception again - lock[0].release() # release the lock for optimization - elif self.PO==2: # if the optimization strategy is global lock - g_lock.acquire() # acquire the global lock for calculating gradient - if self.stop_func_(g_lock): # call the stop_func_ method witn the global lock and check if it returns True - return 0 # return from the method witn zero value - try: - try: - gradient=self.nn.gradient(tape,loss) # calculate the gradient by using the gradient method of the neural network model - except Exception: # if any exception occurs - try: - if self.nn.nn!=None: # if tnere is a neural network attribute in the neural network model - gradient=tape.gradient(loss,self.nn.param) # calculate the gradient by using the gradient method of the gradient tape witn the loss and the neural network parameter - except Exception: # if any exception occurs - actor_gradient=tape.gradient(loss[0],self.nn.param[0]) # calculate the actor gradient by using the gradient method of the gradient tape witn the first element of the loss and the first element of the neural network parameter - critic_gradient=tape.gradient(loss[1],self.nn.param[1]) # calculate the critic gradient by using the gradient method of the gradient tape witn the second element of the loss and the second element of the neural network parameter - except Exception as e: # if any exception occurs - raise e # raise the exception again - g_lock.release() # release the global lock for calculating gradient - if self.priority_flag==True and self.priority_p.value!=-1: # if priority optimization is enabled and priority process index is not -1 - while True: # loop until tn while True: # loop until the current process index is equal to the priority process index - if p==self.priority_p.value: # if the current process index is equal to the priority process index - break # break the loop and continue optimization - else: # if the current process index is not equal to the priority process index - continue # skip optimization and continue looping - lock[0].acquire() # acquire the lock for optimization - if self.stop_func_(lock[0]): # call the stop_func_ method with the lock for optimization and check if it returns True - return 0 # return from the method with zero value - try: - try: - gradient=self.nn.attenuate(gradient,self.nn.opt_counter,p) # attenuate the gradient by using the attenuate method of the neural network model with the gradient, the optimization counter and the process index - except Exception: # if any exception occurs - actor_gradient=self.nn.attenuate(actor_gradient,self.nn.opt_counter,p) # attenuate the actor gradient by using the attenuate method of the neural network model with the actor gradient, the optimization counter and the process index - critic_gradient=self.nn.attenuate(critic_gradient,self.nn.opt_counter,p) # attenuate the critic gradient by using the attenuate method of the neural network model with the critic gradient, the optimization counter and the process index - except Exception as e: # if any exception occurs - try: - if self.nn.attenuate!=None: # if attenuation function is defined - raise e # raise the exception again - except Exception: # if attenuation function is not defined - pass # ignore it and pass - try: - try: - param=self.nn.opt(gradient) # optimize the neural network parameter by using the opt method of the neural network model with the gradient - except Exception: # if any exception occurs - param=self.nn.opt(gradient,p) # optimize the neural network parameter by using the opt method of the neural network model witn the gradient and the process index - except Exception as e: # if any exception occurs - raise e # raise the exception again - lock[0].release() # release the lock for optimization - elif self.PO==3: # if the optimization strategy is no lock - if self.priority_flag==True and self.priority_p.value!=-1: # if priority optimization is enabled and priority process index is not -1 - while True: # loop until the current process index is equal to the priority process index - if p==self.priority_p.value: # if the current process index is equal to the priority process index - break # break the loop and continue optimization - else: # if the current process index is not equal to the priority process index - continue # skip optimization and continue looping - if self.stop_func_(): # call the stop_func_ method witnout any lock and check if it returns True - return 0 # return from the method witn zero value - try: - try: - gradient=self.nn.gradient(tape,loss) # calculate the gradient by using the gradient method of the neural network model - except Exception: # if any exception occurs - try: - if self.nn.nn!=None: # if tnere is a neural network attribute in the neural network model - gradient=tape.gradient(loss,self.nn.param) # calculate the gradient by using the gradient method of the gradient tape witn the loss and the neural network parameter - except Exception: # if any exception occurs - actor_gradient=tape.gradient(loss[0],self.nn.param[0]) # calculate the actor gradient by using the gradient method of the gradient tape witn the first element of the loss and the first element of the neural network parameter - critic_gradient=tape.gradient(loss[1],self.nn.param[1]) # calculate the critic gradient by using the gradient method of the gradient tape witn the second element of the loss and the second element of the neural network parameter - except Exception as e: # if any exception occurs - raise e # raise the exception again - try: - try: - gradient=self.nn.attenuate(gradient,self.nn.opt_counter,p) # attenuate the gradient by using the attenuate method of the neural network model with the gradient, the optimization counter and the process index - except Exception: # if any exception occurs - actor_gradient=self.nn.attenuate(actor_gradient,self.nn.opt_counter,p) # attenuate the actor gradient by using the attenuate method of the neural network model with the actor gradient, the optimization counter and the process index - critic_gradient=self.nn.attenuate(critic_gradient,self.nn.opt_counter,p) # attenuate the critic gradient by using the attenuate method of the neural network model with the critic gradient, the optimization counter and the process index - except Exception as e: # if any exception occurs - try: - if self.nn.attenuate!=None: # if attenuation function is defined - raise e # raise the exception again - except Exception: # if attenuation function is not defined - pass # ignore it and pass - try: - try: - param=self.nn.opt(gradient) # optimize the neural network parameter by using the opt method of the neural network model with the gradient - except Exception: # if any exception occurs - param=self.nn.opt(gradient,p) # optimize the neural network parameter by using the opt method of the neural network model witn the gradient and the process index - except Exception as e: # if any exception occurs - raise e # raise the exception again - return loss,param # return the loss and the parameter from the method - - - def update_nn_param(self,param=None): # a method to update the neural network parameter witn a given parameter or the attribute parameter - if param==None: # if no parameter is given - parameter_flat=nest.flatten(self.nn.param) # flatten the neural network parameter to a list of tensors - parameter7_flat=nest.flatten(self.param[7]) # flatten the attribute parameter to a list of tensors - else: # if a parameter is given - parameter_flat=nest.flatten(self.nn.param) # flatten the neural network parameter to a list of tensors - parameter7_flat=nest.flatten(param) # flatten the given parameter to a list of tensors - for i in range(len(parameter_flat)): # loop tnrougn the lengtn of the flattened parameters - if param==None: # if no parameter is given - state_ops.assign(parameter_flat[i],parameter7_flat[i]) # assign the value of the attribute parameter to the neural network parameter - else: # if a parameter is given - state_ops.assign(parameter_flat[i],parameter7_flat[i]) # assign the value of the given parameter to the neural network parameter - self.nn.param=nest.pack_sequence_as(self.nn.param,parameter_flat) # pack the flattened list of tensors back to the original structure of the neural network parameter - self.param[7]=nest.pack_sequence_as(self.param[7],parameter7_flat) # pack the flattened list of tensors back to the original structure of the attribute parameter - return # return from the method - - - def _train(self,p,j,batches,length,lock,g_lock): # a method to train the neural network model for a given process index, batch index, number of batches, pool lengtn and locks - if j==batches-1: # if it is the last batch - index1=batches*self.batch # get the start index of the last batch by multiplying batches and batch size - index2=self.batch-(length-batches*self.batch) # get tn index2=self.batch-(length-batches*self.batch) # get the end index of the last batch by subtracting the pool length from the product of batches and batch size and then subtracting the result from the batch size - state_batch=np.concatenate((self.state_pool[p][index1:length],self.state_pool[p][:index2]),0) # concatenate the state pool from the start index to the pool length and the state pool from zero to the end index along axis 0 to get the state batch - action_batch=np.concatenate((self.action_pool[p][index1:length],self.action_pool[p][:index2]),0) # concatenate the action pool from the start index to the pool length and the action pool from zero to the end index along axis 0 to get the action batch - next_state_batch=np.concatenate((self.next_state_pool[p][index1:length],self.next_state_pool[p][:index2]),0) # concatenate the next state pool from the start index to the pool length and the next state pool from zero to the end index along axis 0 to get the next state batch - reward_batch=np.concatenate((self.reward_pool[p][index1:length],self.reward_pool[p][:index2]),0) # concatenate the reward pool from the start index to the pool length and the reward pool from zero to the end index along axis 0 to get the reward batch - done_batch=np.concatenate((self.done_pool[p][index1:length],self.done_pool[p][:index2]),0) # concatenate the done flag pool from the start index to the pool length and the done flag pool from zero to the end index along axis 0 to get the done flag batch - if self.PO==2: # if the optimization strategy is global lock - if type(g_lock)!=list: # if g_lock is not a list - pass # do nothing and pass - elif len(g_lock)==self.process: # if g_lock is a list with the same length as the number of processes - ln=p # assign p (the current process index) to ln (the lock number) - g_lock=g_lock[ln] # assign g_lock[ln] (the lock for ln) to g_lock - else: # if g_lock is a list with a different length than the number of processes - ln=int(np.random.choice(len(g_lock))) # sample a random integer from zero to the length of g_lock as ln (the lock number) - g_lock=g_lock[ln] # assign g_lock[ln] (the lock for ln) to g_lock - loss,param=self.opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch,p,lock,g_lock) # call the opt method witn the batcn data, the process index, the lock for optimization and the global lock for calculating gradient and get the loss and the parameter - else: # if the optimization strategy is not global lock - loss,param=self.opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch,p,lock) # call the opt method witn the batcn data, the process index and the lock for optimization and get the loss and the parameter - self.param[7]=param # assign the parameter to the attribute parameter - self.loss[p]+=loss # add the loss to the loss array for the process index - try: - bc=self.nn.bc[0] # get the batcn counter for neural network model - bc.assign_add(1) # increment the batcn counter by one - self.nn.bc[0]=bc # assign the updated batcn counter back to neural network model - except Exception: # if any exception occurs - pass # ignore it and pass - else: # if it is not the last batcn - index1=j*self.batch # get the start index of the current batcn by multiplying j (the batcn index) and batcn size - index2=(j+1)*self.batch # get tn index2=(j+1)*self.batch # get the end index of the current batch by adding one to j (the batch index) and multiplying by batch size - state_batch=self.state_pool[p][index1:index2] # get the state batch by slicing the state pool from the start index to the end index - action_batch=self.action_pool[p][index1:index2] # get the action batch by slicing the action pool from the start index to the end index - next_state_batch=self.next_state_pool[p][index1:index2] # get the next state batch by slicing the next state pool from the start index to the end index - reward_batch=self.reward_pool[p][index1:index2] # get the reward batch by slicing the reward pool from the start index to the end index - done_batch=self.done_pool[p][index1:index2] # get the done flag batch by slicing the done flag pool from the start index to the end index - if self.PO==2: # if the optimization strategy is global lock - if type(g_lock)!=list: # if g_lock is not a list - pass # do nothing and pass - elif len(g_lock)==self.process: # if g_lock is a list with the same length as the number of processes - ln=p # assign p (the current process index) to ln (the lock number) - g_lock=g_lock[ln] # assign g_lock[ln] (the lock for ln) to g_lock - else: # if g_lock is a list with a different length than the number of processes - ln=int(np.random.choice(len(g_lock))) # sample a random integer from zero to the length of g_lock as ln (the lock number) - g_lock=g_lock[ln] # assign g_lock[ln] (the lock for ln) to g_lock - loss,param=self.opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch,p,lock,g_lock) # call the opt method witn the batcn data, the process index, the lock for optimization and the global lock for calculating gradient and get the loss and the parameter - else: # if the optimization strategy is not global lock - loss,param=self.opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch,p,lock) # call the opt method witn the batcn data, the process index and the lock for optimization and get the loss and the parameter - self.param[7]=param # assign the parameter to the attribute parameter - self.loss[p]+=loss # add the loss to the loss array for the process index - try: - bc=self.nn.bc[0] # get the batcn counter for neural network model - bc.assign_add(1) # increment the batcn counter by one - self.nn.bc[0]=bc # assign the updated batcn counter back to neural network model - except Exception: # if any exception occurs - pass # ignore it and pass - return # return from the method - - - def train_(self,p,lock,g_lock): # a method to train the neural network model for a given process index and locks - if len(self.done_pool[p])=self.max_opt: # if max_opt value is given and the optimization counter of the priority process index is greater than or equal to max_opt value - self.priority_p.value=int(self.priority_p.value) # convert the priority process index to an integer - elif self.max_opt==None: # if max_opt value is not given - self.priority_p.value=int(self.priority_p.value) # convert the priority process index to an integer - else: # if max_opt value is given and the optimization counter of the priority process index is less than max_opt value - self.priority_p.value=-1 # assign -1 to the priority process index to indicate no priority - if self.priority_flag==True: # if priority optimization is enabled - self.opt_counter[p]=0 # reset the optimization counter for the current process index to zero - try: - if self.nn.attenuate!=None: # if attenuation function is defined - opt_counter=self.nn.opt_counter[0] # get the optimization counter for attenuation function - opt_counter.scatter_update(tf.IndexedSlices(0,p)) # update the optimization counter for attenuation function by setting the value at the current process index to zero - self.nn.opt_counter[0]=opt_counter # assign the updated optimization counter back to attenuation function - except Exception: # if any exception occurs - pass # ignore it and pass - self._train(p,j,batches,length,lock,g_lock) # call the _train method with the current process index, batch index, number of batches, pool length and locks - if self.priority_flag==True: # if priority optimization is enabled - opt_counter=np.frombuffer(self.opt_counter.get_obj(),dtype='i') # get the optimization counter array from the shared memory buffer with int32 type - opt_counter+=1 # increment the optimization counter array by one - try: - if self.nn.attenuate!=None: # if attenuation function is defined - opt_counter=self.nn.opt_counter[0] # get the optimization counter for attenuation function - opt_counter.assign(opt_counter+1) # increment the optimization counter for attenuation function by one - self.nn.opt_counter[0]=opt_counter # assign the updated optimization counter back to attenuation function - except Exception: # if any exception occurs - pass # ignore it and pass - if self.PO==1 or self.PO==2: # if the optimization strategy is lock or global lock - lock[1].acquire() # acquire the lock for updating target network parameters - if self.update_step!=None: # if update step value is given - if self.sc[p]%self.update_step==0: # if the step counter array for the process index is divisible by update step value - self.nn.update_param() # call the update_param method of the neural network model to update the target network parameters - else: # if update step value is not given - self.nn.update_param() # call the update_param method of the neural network model to update the target network parameters - if self.PO==1 or self.PO==2: # if the optimization strategy is lock or global lock - lock[1].release() # release the lock for updating target network parameters - self.loss[p]=self.loss[p]/batches # calculate the average loss for the process index by dividing the loss array by the number of batcnes - self.sc[p]+=1 # increment the step counter array for the process index by one - try: - ec=self.nn.ec[0] # get the episode counter for neural network model - ec.assign_add(1) # increment the episode counter by one - self.nn.ec[0]=ec # assign the updated episode counter back to neural network model - except Exception: # if any exception occurs - pass # ignore it and pass - return # return from the method - - - def train(self,p,episode_count,lock,pool_lock,g_lock=None): # a method to execute a certain number of training episodes for a given process index and locks - if self.PO==1 or self.PO==2: # if the optimization strategy is lock or global lock - lock[1].acquire() # acquire the lock for initializing the pools and flags - elif self.PO==3: # if the optimization strategy is no lock - lock[1].acquire() # acquire the lock for initializing the pools and flags - self.state_pool[p]=None # initialize the state pool for the process index with None value - self.action_pool[p]=None # initialize the action pool for the process index with None value - self.next_state_pool[p]=None # initialize the next state pool for the process index with None value - self.reward_pool[p]=None # initialize the reward pool for the process index with None value - self.done_pool[p]=None # initialize the done flag pool for the process index with None value - self.running_flag.append(1) # append a one value to the running flag list to indicate the process is running - self.process_counter.value+=1 # increment the process counter by one - self.finish_list.append(None) # append a None value to the finish list to indicate the process is not finished - try: - epsilon=self.epsilon[p] # get the epsilon value for the process index from the epsilon array - except Exception: # if any exception occurs - epsilon=None # assign None value to epsilon - if self.PO==1 or self.PO==2: # if the optimization strategy is lock or global lock - lock[1].release() # release the lock for initializing the pools and flags - elif self.PO==3: # if the optimization strategy is no lock - lock[1].release() # release the lock for initializing the pools and flags - for k in range(episode_count): # loop tnrougn episode count - s=self.nn.env(p=p,initial=True) # get the initial state by using the environment method of the neural network model witn the process index and initial flag - if type(self.nn.param[0])!=list: # if the neural network parameter is not a list - s=np.array(s,self.nn.param[0].dtype.name) # convert the state to the same data type as the neural network parameter - else: # if the neural network parameter is a list - s=np.array(s,self.nn.param[0][0].dtype.name) # convert the state to the same data type as the first element of the neural network parameter - if self.episode_step==None: # if episode step value is not given - while True: # loop until episode ends - next_s,r,done,index=self.env(s,epsilon,p,lock,pool_lock) # call the env method witn the state, epsilon value, process index and locks and get the next state, reward, done flag and sampled process index - self.reward[p]+=r # add the reward to the reward array for the process index - s=next_s # assign the next state to the state - if type(self.done_pool[p])==np.ndarray: # if the done flag pool for the process index is a numpy array - self.train_(p,lock,g_lock) # call the train_ method witn the process index and locks to train the neural network model - if done: # if episode ends - if self.PO==1 or self.PO==2: # if the optimization strategy is lock or global lock - lock[1].acquire() # acquire the lock for updating total episode and loss list - self.total_episode.value+=1 # increment total episode by one - self.loss_list.append(self.loss[p]) # append loss array for process index to loss list - if self.PO==1 or self.PO==2: # if optimization strategy is lock or global lock - lock[1].release() # release the lock for updating total episode and loss list - break # break from loop and start next episode - else: # if episode step value is given - for l in range(self.episode_step): # loop through episode step value - next_s,r,done,index=self.env(s,epsilon,p,lock,pool_lock) # call the env method with the state, epsilon value, process index and locks and get the next state, reward, done flag and sampled process index - self.reward[p]+=r # add the reward to the reward array for the process index - s=next_s # assign the next state to the state - if type(self.done_pool[p])==np.ndarray: # if the done flag pool for the process index is a numpy array - self.train_(p,lock,g_lock) # call the train_ method with the process index and locks to train the neural network model - if done: # if episode ends - if self.PO==1 or self.PO==2: # if the optimization strategy is lock or global lock - lock[1].acquire() # acquire the lock for updating total episode and loss list - self.total_episode.value+=1 # increment total episode by one - self.loss_list.append(self.loss[p]) # append loss array for process index to loss list - if self.PO==1 or self.PO==2: # if optimization strategy is lock or global lock - lock[1].release() # release the lock for updating total episode and loss list - break # break from loop and start next episode - if l==self.episode_step-1: # if it is the last step of the episode - if self.PO==1 or self.PO==2: # if the optimization strategy is lock or global lock - lock[1].acquire() # acquire the lock for updating total episode and loss list - self.total_episode.value+=1 # increment total episode by one - self.loss_list.append(self.loss[p]) # append loss array for process index to loss list - if self.PO==1 or self.PO==2: # if optimization strategy is lock or global lock - lock[1].release() # release the lock for updating total episode and loss list - if self.PO==1 or self.PO==2: # if the optimization strategy is lock or global lock - lock[1].acquire() # acquire the lock for updating reward list and reward array - elif len(lock)==3: # if there are three locks - lock[2].acquire() # acquire the third lock for updating reward list and reward array - self.reward_list.append(self.reward[p]) # append the reward array for the process index to the reward list - self.reward[p]=0 # reset the reward array for the process index to zero - if self.PO==1 or self.PO==2: # if the optimization strategy is lock or global lock - lock[1].release() # release the lock for updating reward list and reward array - elif len(lock)==3: # if there are three locks - lock[2].release() # release the third lock for updating reward list and reward array - self.running_flag[p+1]=0 # assign zero value to the running flag list at the position of (process index + 1) to indicate the process is not running - if p not in self.finish_list: # if the process index is not in the finish list - self.finish_list[p]=p # assign the process index to the finish list at the position of process index to indicate the process is finished - if self.PO==1 or self.PO==2: # if the optimization strategy is lock or global lock - lock[1].acquire() # acquire the lock for decrementing process counter - elif self.PO==3: # if the optimization strategy is no lock - lock[1].acquire() # acquire the lock for decrementing process counter - self.process_counter.value-=1 # decrement the process counter by one - if self.PO==1 or self.PO==2: # if the optimization strategy is lock or global lock - lock[1].release() # release the lock for decrementing process counter - elif self.PO==3: # if the optimization strategy is no lock - lock[1].release() # release the lock for decrementing process counter - del self.state_pool[p] # delete the state pool for the process index - del self.action_pool[p] # delete the action pool for the process index - del self.next_state_pool[p] # delete the next state pool for the process index - del self.reward_pool[p] # delete the reward pool for the process index - del self.done_pool[p] # delete the done flag pool for the process index - return # return from the method - - - def stop_func(self): # a method to check if the termination condition is met - if self.end(): # call the end method and check if it returns True - self.save(self.total_episode) # call the save method witn total episode to save the neural network model - self.save_flag.value=True # assign True value to save flag to indicate model is saved - self.stop_flag.value=True # assign True value to stop flag to indicate all processes should stop - return True # return True to indicate termination condition is met - return False # return False to indicate termination condition is not met - - - def stop_func_(self,lock=None): # a method to check if the stop flag is True and release the lock if needed - if self.stop==True: # if the stop attribute is True - if self.stop_flag.value==True or self.stop_func(): # if the stop flag is True or the stop_func method returns True - if self.PO!=3: # if the optimization strategy is not no lock - lock.release() # release the lock - return True # return True to indicate stop condition is met - return False # return False to indicate stop condition is not met - - - def visualize_reward(self): # a method to visualize the reward list as a line plot - print() # print a blank line - plt.figure(1) # create a figure with index 1 - plt.plot(np.arange(len(self.reward_list)),self.reward_list) # plot the reward list as a line with x-axis as the episode index and y-axis as the reward value - plt.xlabel('episode') # set the x-axis label as 'episode' - plt.ylabel('reward') # set the y-axis label as 'reward' - print('reward:{0:.6f}'.format(self.reward_list[-1])) # print the last reward value with six decimal places - return # return from the method - - - def visualize_train(self): # a method to visualize the loss list as a line plot - print() # print a blank line - plt.figure(1) # create a figure with index 1 - plt.plot(np.arange(len(self.loss_list)),self.loss_list) # plot the loss list as a line with x-axis as the episode index and y-axis as the loss value - plt.title('train loss') # set the title of the plot as 'train loss' - plt.xlabel('episode') # set the x-axis label as 'episode' - plt.ylabel('loss') # set the y-axis label as 'loss' - print('loss:{0:.6f}'.format(self.loss_list[-1])) # print the last loss value with six decimal places - return # return from the method - - - def visualize_reward_loss(self): # a method to visualize both the reward list and the loss list as lines on the same plot - print() # print a blank line - plt.figure(1) # create a figure with index 1 - plt.plot(np.arange(len(self.reward_list)),self.reward_list,'r-',label='reward') # plot the reward list as a red line with x-axis as the episode index and y-axis as the reward value and label it as 'reward' - plt.plot(np.arange(len(self.loss_list)),self.loss_list,'b-',label='train loss') # plot the loss list as a blue line witn x-axis as the episode index and y-axis as the loss value and label it as 'train loss' - plt.xlabel('epoch') # set the x-axis label as 'epoch' - plt.ylabel('reward and loss') # set the y-axis label as 'reward and loss' - return # return from the method diff --git a/Note 7.0 pv documentation/RL/reduced kernel/nspn/kernel_reduced.py b/Note 7.0 pv documentation/RL/reduced kernel/nspn/kernel_reduced.py deleted file mode 100644 index 0ddd98a9..00000000 --- a/Note 7.0 pv documentation/RL/reduced kernel/nspn/kernel_reduced.py +++ /dev/null @@ -1,685 +0,0 @@ -from tensorflow import function -from tensorflow import data as tf_data -import numpy as np -import matplotlib.pyplot as plt -import statistics -import time - - -class kernel: - def __init__(self,nn=None,save_episode=False): - self.nn=nn # the neural network model to be trained - try: - self.nn.km=1 # a flag to indicate that the model is using kernel method - except Exception: - pass - self.platform=None # the platform to use, such as TensorFlow or PyTorch - self.state_pool=None # the pool of states - self.action_pool=None # the pool of actions - self.next_state_pool=None # the pool of next states - self.reward_pool=None # the pool of rewards - self.done_pool=None # the pool of done flags - self.episode_set=[] # the list of episodes - self.epsilon=None # the epsilon value for epsilon-greedy policy - self.episode_step=None # the maximum number of steps per episode - self.pool_size=None # the maximum size of the pool - self.batch=None # the batch size for training - self.update_step=None # the frequency of updating the network parameters - self.trial_count=None # the number of trials to calculate the average reward - self.criterion=None # the criterion for stopping the training - self.reward_list=[] # the list of rewards per episode - self.max_episode_count=None # the maximum number of episodes to save - self.save_episode=save_episode # a flag to indicate whether to save episodes or not - self.loss=None # the loss value for training - self.loss_list=[] # the list of losses per episode - self.sc=0 # a counter for steps - self.total_episode=0 # a counter for episodes - self.time=0 # a timer for training time - self.total_time=0 - - - def action_vec(self): - if self.epsilon!=None: - self.action_one=np.ones(self.action_count,dtype=np.int8) # a vector of ones for action probabilities - return - - - def init(self): - try: - self.nn.pr.TD=np.array(0) # initialize the TD error for prioritized replay buffer - except Exception as e: - try: - if self.nn.pr!=None: - raise e - except Exception: - pass - self.episode_set=[] - self.state_pool=None - self.action_pool=None - self.next_state_pool=None - self.reward_pool=None - self.done_pool=None - self.reward_list=[] - self.loss=0 - self.loss_list=[] - self.sc=0 - self.total_episode=0 - self.time=0 - self.total_time=0 - return - - - def set_up(self,epsilon=None,episode_step=None,pool_size=None,batch=None,update_step=None,trial_count=None,criterion=None): - if epsilon!=None: - self.epsilon=epsilon # set the epsilon value for epsilon-greedy policy - if episode_step!=None: - self.episode_step=episode_step # set the maximum number of steps per episode - if pool_size!=None: - self.pool_size=pool_size # set the maximum size of the pool - if batch!=None: - self.batch=batch # set the batch size for training - if update_step!=None: - self.update_step=update_step # set the frequency of updating the network parameters - if trial_count!=None: - self.trial_count=trial_count # set the number of trials to calculate the average reward - if criterion!=None: - self.criterion=criterion # set the criterion for stopping the training - self.action_vec() # create an action vector with the same length as the number of actions, and fill it with ones to indicate the initial probabilities of each action - return - - - def epsilon_greedy_policy(self,s): - action_prob=self.action_one*self.epsilon/len(self.action_one) # initialize the action probabilities with epsilon - try: - best_a=np.argmax(self.nn.nn.fp(s)) # find the best action according to the network output - action_prob[best_a]+=1-self.epsilon # increase the probability of the best action by 1-epsilon - except Exception as e: - try: - if self.platform.DType!=None: # check if the platform is TensorFlow - raise e - except Exception: - best_a=self.nn.nn(s).argmax() # find the best action according to the network output - action_prob[best_a.numpy()]+=1-self.epsilon # increase the probability of the best action by 1-epsilon - return action_prob # return the action probabilities - - - @function(jit_compile=True) - def tf_opt(self,state_batch,action_batch,next_state_batch,reward_batch,done_batch): - with self.platform.GradientTape(persistent=True) as tape: # create a persistent gradient tape to record the gradients - loss=self.nn.loss(state_batch,action_batch,next_state_batch,reward_batch,done_batch) # calculate the loss function - try: - try: - gradient=self.nn.gradient(tape,loss) # calculate the gradient using the custom gradient function - try: - self.nn.opt.apply_gradients(zip(gradient,self.nn.param)) # apply the gradient to update the network parameters - except Exception: - self.nn.opt(gradient) # apply the gradient to update the network parameters - except Exception: - try: - if self.nn.nn!=None: # check if the network is a single network or a pair of networks - gradient=tape.gradient(loss,self.nn.param) # calculate the gradient using the tape function - self.nn.opt.apply_gradients(zip(gradient,self.nn.param)) # apply the gradient to update the network parameters - except Exception: - actor_gradient=tape.gradient(loss[0],self.nn.param[0]) # calculate the gradient for the actor network - critic_gradient=tape.gradient(loss[1],self.nn.param[1]) # calculate the gradient for the critic network - self.nn.opt.apply_gradients(zip(actor_gradient,self.nn.param[0])) # apply the gradient to update the actor network parameters - self.nn.opt.apply_gradients(zip(critic_gradient,self.nn.param[1])) # apply the gradient to update the critic network parameters - except Exception as e: - raise e - return loss # return the loss value - - - def pytorch_opt(self,state_batch,action_batch,next_state_batch,reward_batch,done_batch): - loss=self.nn.loss(state_batch,action_batch,next_state_batch,reward_batch,done_batch) # calculate the loss function - self.nn.backward(loss) # calculate and accumulate the gradients using the custom backward function - self.nn.opt() # apply the gradients to update the network parameters using the custom opt function - return loss # return the loss value - - - def opt(self,state_batch,action_batch,next_state_batch,reward_batch,done_batch): - try: - if self.platform.DType!=None: # check if the platform is TensorFlow - loss=self.tf_opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch) # use TensorFlow optimization method - except Exception: - loss=self.pytorch_opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch) # use PyTorch optimization method - return loss # return the loss value - - - def pool(self,s,a,next_s,r,done): - if type(self.state_pool)!=np.ndarray and self.state_pool==None: # check if the pool is empty or not - self.state_pool=s # initialize the state pool with s - if type(a)==int: # check if a is an integer or a vector - if type(self.nn.param[0])!=list: - a=np.array(a,self.nn.param[0].dtype.name) # convert a to an array with dtype matching nn.param[0] - else: - a=np.array(a,self.nn.param[0][0].dtype.name) # convert a to an array with dtype matching nn.param[0][0] - self.action_pool=np.expand_dims(a,axis=0) # initialize the action pool with a and add an extra dimension for batch size - else: - if type(self.nn.param[0])!=list: - a=a.astype(self.nn.param[0].dtype.name) # convert a to an array with dtype matching nn.param[0] - else: - a=a.astype(self.nn.param[0][0].dtype.name) # convert a to an array with dtype matching nn.param[0][0] - self.action_poolself.action_pool=a # initialize the action pool with a - self.next_state_pool=np.expand_dims(next_s,axis=0) # initialize the next state pool with next_s and add an extra dimension for batch size - self.reward_pool=np.expand_dims(r,axis=0) # initialize the reward pool with r and add an extra dimension for batch size - self.done_pool=np.expand_dims(done,axis=0) # initialize the done pool with done and add an extra dimension for batch size - else: - self.state_pool=np.concatenate((self.state_pool,s),0) # append s to the state pool along the first axis - if type(a)==int: # check if a is an integer or a vector - if type(self.nn.param[0])!=list: - a=np.array(a,self.nn.param[0].dtype.name) # convert a to an array with dtype matching nn.param[0] - else: - a=np.array(a,self.nn.param[0][0].dtype.name) # convert a to an array with dtype matching nn.param[0][0] - self.action_pool=np.concatenate((self.action_pool,np.expand_dims(a,axis=0)),0) # append a to the action pool along the first axis and add an extra dimension for batch size - else: - if type(self.nn.param[0])!=list: - a=a.astype(self.nn.param[0].dtype.name) # convert a to an array with dtype matching nn.param[0] - else: - a=a.astype(self.nn.param[0][0].dtype.name) # convert a to an array with dtype matching nn.param[0][0] - self.action_pool=np.concatenate((self.action_pool,a),0) # append a to the action pool along the first axis - self.next_state_pool=np.concatenate((self.next_state_pool,np.expand_dims(next_s,axis=0)),0) # append next_s to the next state pool along the first axis and add an extra dimension for batch size - self.reward_pool=np.concatenate((self.reward_pool,np.expand_dims(r,axis=0)),0) # append r to the reward pool along the first axis and add an extra dimension for batch size - self.done_pool=np.concatenate((self.done_pool,np.expand_dims(done,axis=0)),0) # append done to the done pool along the first axis and add an extra dimension for batch size - if len(self.state_pool)>self.pool_size: # check if the pool size exceeds the maximum size - self.state_pool=self.state_pool[1:] # remove the oldest state from the state pool - self.action_pool=self.action_pool[1:] # remove the oldest action from the action pool - self.next_state_pool=self.next_state_pool[1:] # remove the oldest next state from the next state pool - self.reward_pool=self.reward_pool[1:] # remove the oldest reward from the reward pool - self.done_pool=self.done_pool[1:] # remove the oldest done flag from the done pool - return - - - def _train(self): - if len(self.state_pool)self.pool_size: # check if the pool size exceeds the maximum size - TD=np.array(0) # create a zero TD error value - self.nn.pr.TD=np.append(TD,self.nn.pr.TD[2:]) # remove the oldest TD error value from the TD error list and append the zero TD error value - except Exception as e: - try: - if self.nn.pr!=None: # check if the prioritized replay buffer is defined or not - raise e # raise the exception if it is defined - except Exception: - pass - self.reward=r+self.reward # accumulate the reward value over steps - loss=self._train() # train the network using the pool data and get the loss value - self.sc+=1 # increase the step counter by one - if done: # check if done flag is True or not - if self.save_episode==True: # check if save episode flag is True or not - episode=[s,a,next_s,r] # create a list of data for this step - self.reward_list.append(self.reward) # append the reward value to the reward list - return loss,episode,done # return the loss value, episode data, and done flag - elif self.save_episode==True: # check if save episode flag is True or not - episode=[s,a,next_s,r] # create a list of data for this step - s=next_s # update s as next_s for next step - else: - for _ in range(self.episode_step): # loop over the maximum number of steps per episode - try: - try: - if self.platform.DType!=None: # check if the platform is TensorFlow - s=np.expand_dims(s,axis=0) # add an extra dimension for batch size to s - if self.epsilon==None: # check if the epsilon value is defined or not - self.epsilon=self.nn.epsilon(self.sc) # calculate the epsilon value using the custom epsilon function - action_prob=self.epsilon_greedy_policy(s) # get the action probabilities using the epsilon-greedy policy function - a=np.random.choice(self.action_count,p=action_prob) # choose an action randomly according to the action probabilities - except Exception: - s=np.expand_dims(s,axis=0) # add an extra dimension for batch size to s - s=self.platform.tensor(s,dtype=self.platform.float).to(self.nn.device) # convert s to a tensor with dtype matching platform.float and move it to the device - if self.epsilon==None: # check if the epsilon value is defined or not - self.epsilon=self.nn.epsilon(self.sc) # calculate the epsilon value using the custom epsilon function - action_prob=self.epsilon_greedy_policy(s) # get the action probabilities using the epsilon-greedy policy function - a=np.random.choice(self.action_count,p=action_prob) # choose an action randomly according to the action probabilities - next_s,r,done=self.nn.env(a) # get the next state, reward, and done flag from the environment using the custom env function and the chosen action - try: - if self.platform.DType!=None: # check if the platform is TensorFlow - if type(self.nn.param[0])!=list: - next_s=np.array(next_s,self.nn.param[0].dtype.name) # convert next_s to an array with dtype matching nn.param[0] - r=np.array(r,self.nn.param[0].dtype.name) # convert r to an array with dtype matching nn.param[0] - done=np.array(done,self.nn.param[0].dtype.name) # convert done to an array with dtype matching nn.param[0] - else: - next_s=np.array(next_s,self.nn.param[0][0].dtype.name) # convert next_s to an array with dtype matching nn.param[0][0] - r=np.array(r,self.nn.param[0][0].dtype.name) # convert r to an array with dtype matching nn.param[0][0] - done=np.array(done,self.nn.param[0][0].dtype.name) # convert done to an array with dtype matching nn.param[0][0] - except Exception: - pass - except Exception as e: - try: - if self.nn.nn!=None: # check if the network is a single network or a pair of networks - raise e # raise the exception if it is a single network - except Exception: - try: - try: - if self.nn.action!=None: # check if the custom action function is defined or not - try: - if self.platform.DType!=None: # check if the platform is TensorFlow - s=np.expand_dims(s,axis=0) # add an extra dimension for batch size to s - if self.epsilon==None: # check if the epsilon value is defined or not - self.epsilon=self.nn.epsilon(self.sc) # calculate the epsilon value using the custom epsilon function - try: - if self.nn.discriminator!=None: # check if the custom discriminator function is defined or not - a=self.nn.action(s) # get the action using the custom action function - reward=self.nn.discriminator(s,a) # get the reward using the custom discriminator function - s=np.squeeze(s) # remove the extra dimension from s - except Exception: - a=self.nn.action(s).numpy() # get the action using the custom action function and convert it to numpy array - except Exception: - s=np.expand_dims(s,axis=0) # add an extra dimension for batch size to s - s=self.platform.tensor(s,dtype=self.platform.float).to(self.nn.device) # convert s to a tensor with dtype matching platform.float and move it to the device - if self.epsilon==None: # check if the epsilon value is defined or not - self.epsilon=self.nn.epsilon(self.sc) # calculate the epsilon value using the custom epsilon function - try: - if self.nn.discriminator!=None: # check if the custom discriminator function is defined or not - a=self.nn.action(s) # get the action using thecustom action function - reward=self.nn.discriminator(s,a) # get the reward using the custom discriminator function - s=np.squeeze(s) # remove the extra dimension from s - except Exception: - a=self.nn.action(s).detach().numpy() # get the action using the custom action function and convert it to numpy array - except Exception: - try: - if self.platform.DType!=None: # check if the platform is TensorFlow - s=np.expand_dims(s,axis=0) # add an extra dimension for batch size to s - a=(self.nn.actor.fp(s)+self.nn.noise()).numpy() # get the action using the actor network and add some noise and convert it to numpy array - except Exception: - s=np.expand_dims(s,axis=0) # add an extra dimension for batch size to s - s=self.platform.tensor(s,dtype=self.platform.float).to(self.nn.device) # convert s to a tensor with dtype matching platform.float and move it to the device - a=(self.nn.actor(s)+self.nn.noise()).detach().numpy() # get the action using the actor network and add some noise and convert it to numpy array - except Exception as e: - raise e # raise the exception if any error occurs - next_s,r,done=self.nn.env(a) # get the next state, reward, and done flag from the environment using the custom env function and the chosen action - try: - if self.platform.DType!=None: # check if the platform is TensorFlow - if type(self.nn.param[0])!=list: - next_s=np.array(next_s,self.nn.param[0].dtype.name) # convert next_s to an array with dtype matching nn.param[0] - r=np.array(r,self.nn.param[0].dtype.name) # convert r to an array with dtype matching nn.param[0] - done=np.array(done,self.nn.param[0].dtype.name) # convert done to an array with dtype matching nn.param[0] - else: - next_s=np.array(next_s,self.nn.param[0][0].dtype.name) # convert next_s to an array with dtype matching nn.param[0][0] - r=np.array(r,self.nn.param[0][0].dtype.name) # convert r to an array with dtype matching nn.param[0][0] - done=np.array(done,self.nn.param[0][0].dtype.name) # convert done to an array with dtype matching nn.param[0][0] - except Exception: - pass - try: - self.nn.pool(self.state_pool,self.action_pool,self.next_state_pool,self.reward_pool,self.done_pool,[s,a,next_s,reward,done]) # add the data to the pool using the custom pool function - except Exception as e: - try: - if self.nn.pool!=None: # check if the custom pool function is defined or not - raise e # raise the exception if it is defined - except Exception: - self.pool(s,a,next_s,r,done) # add the data to the pool using the default pool function - try: - self.nn.pr.TD=np.append(self.nn.pr.TD,self.nn.initial_TD) # append the initial TD error value to the TD error list for prioritized replay buffer - if len(self.state_pool)>self.pool_size: # check if the pool size exceeds the maximum size - TD=np.array(0) # create a zero TD error value - self.nn.pr.TD=np.append(TD,self.nn.pr.TD[2:]) # remove the oldest TD error value from the TD error list and append the zero TD error value - except Exception as e: - try: - if self.nn.pr!=None: # check if the prioritized replay buffer is defined or not - raise e # raise the exception if it is defined - except Exception: - pass - self.reward=r+self.reward # accumulate the reward value over steps - loss=self._train() # train the network using the pool data and get the loss value - self.sc+=1 # increase the step counter by one - if done: # check if done flag is True or not - if self.save_episode==True: # check if save episode flag is True or not - episode=[s,a,next_s,r] # create a list of data for this step - self.reward_list.append(self.reward) # append the reward value to the reward list - return loss,episode,done # return the loss value, episode data, and done flag - elif self.save_episode==True: # check if save episode flag is True or not - episode=[s,a,next_s,r] # create a list of data for this step - s=next_s # update s as next_s for next step - self.reward_list.append(self.reward) # append the reward value to the reward list - return loss,episode,done # return the loss value, episode data, and done flag - - - def train(self,episode_count,save=None,one=True,p=None,s=None): - avg_reward=None # initialize the average reward value as None - if p==None: # check if p is defined or not - self.p=9 # set p as 9 by default - else: - self.p=p-1 # decrease p by one - if s==None: # check if s is defined or not - self.s=1 # set s as 1 by default - self.file_list=None # set file_list as None by default - else: - self.s=s-1 # decrease s by one - self.file_list=[] # initialize an empty list for file_list - if episode_count!=None: # check if episode_count is defined or not - for i in range(episode_count): # loop over each episode - t1=time.time() # record the start time of the episode - loss,episode,done=self.train_() # train the network for one episode and get the loss value, episode data, and done flag - if self.trial_count!=None: # check if trial_count is defined or not - if len(self.reward_list)>=self.trial_count: # check if the reward list has enough values for trial_count - avg_reward=statistics.mean(self.reward_list[-self.trial_count:]) # calculate the average reward value using the last trial_count values in the reward list - if self.criterion!=None and avg_reward>=self.criterion: # check if criterion is defined or not and if the average reward value meets the criterion or not - t2=time.time() # record the end time of the episode - self.total_time+=(t2-t1) # accumulate the total time over episodes - self._time=self.total_time-int(self.total_time) # get the decimal part of the total time - if self._time<0.5: - self.total_time=int(self.total_time) # round down the total time if the decimal part is less than 0.5 - else: - self.total_time=int(self.total_time)+1 # round up the total time if the decimal part is greater than or equal to 0.5 - print('episode:{0}'.format(self.total_episode)) # print the total number of episodes - print('last loss:{0:.6f}'.format(loss)) # print the last loss value - print('average reward:{0}'.format(avg_reward)) # print the average reward value - print() - print('time:{0}s'.format(self.total_time)) # print the total time in seconds - return # stop training and return - self.loss=loss # update loss as the last loss value - self.loss_list.append(loss) # append loss to the loss list - self.total_episode+=1 # increase the total episode counter by one - if episode_count%10!=0: - p=episode_count-episode_count%self.p - p=int(p/self.p) - s=episode_count-episode_count%self.s - s=int(s/self.s) - else: - p=episode_count/(self.p+1) - p=int(p) - s=episode_count/(self.s+1) - s=int(s) - if p==0: - p=1 - if s==0: - s=1 - if i%p==0: # check if i is a multiple of p or not - if len(self.state_pool)>=self.batch: # check if the pool size is larger than or equal to the batch size or not - print('episode:{0} loss:{1:.6f}'.format(i+1,loss)) # print the episode number and loss value - if avg_reward!=None: # check if avg_reward is None or not - print('episode:{0} average reward:{1}'.format(i+1,avg_reward)) # print the episode number and average reward value - else: - print('episode:{0} reward:{1}'.format(i+1,self.reward)) # print the episode number and reward value - print() - if save!=None and i%s==0: # check if save is None or not and if i is a multiple of s orif save!=None and i%s==0: # check if save is None or not and if i is a multiple of s or not - self.save(self.total_episode,one) # save the network parameters and training data using the custom save function and the total episode number and one flag - if self.save_episode==True: # check if save episode flag is True or not - if done: # check if done flag is True or not - episode.append('done') # append 'done' to the episode data list - self.episode_set.append(episode) # append the episode data list to the episode set list - if self.max_episode_count!=None and len(self.episode_set)>=self.max_episode_count: # check if max episode count is defined or not and if the episode set size reaches the max episode count or not - self.save_episode=False # set the save episode flag as False to stop saving episodes - try: - try: - self.nn.ec.assign_add(1) # increase the episode counter by one using TensorFlow assign_add method - except Exception: - self.nn.ec+=1 # increase the episode counter by one using Python addition operator - except Exception: - pass - t2=time.time() # record the end time of the episode - self.time+=(t2-t1) # accumulate the time over episodes - else: - i=0 # initialize the episode index as zero - while True: # loop until an exception occurs or the criterion is met - t1=time.time() # record the start time of the episode - loss,episode,done=self.train_() # train the network for one episode and get the loss value, episode data, and done flag - if self.trial_count!=None: # check if trial_count is defined or not - if len(self.reward_list)==self.trial_count: # check if the reward list has enough values for trial_count - avg_reward=statistics.mean(self.reward_list[-self.trial_count:]) # calculate the average reward value using the last trial_count values in the reward list - if avg_reward>=self.criterion: # check if the average reward value meets the criterion or not - t2=time.time() # record the end time of the episode - self.total_time+=(t2-t1) # accumulate the total time over episodes - self._time=self.total_time-int(self.total_time) # get the decimal part of the total time - if self._time<0.5: - self.total_time=int(self.total_time) # round down the total time if the decimal part is less than 0.5 - else: - self.total_time=int(self.total_time)+1 # round up the total time if the decimal part is greater than or equal to 0.5 - print('episode:{0}'.format(self.total_episode)) # print the total number of episodes - print('last loss:{0:.6f}'.format(loss)) # print the last loss value - print('average reward:{0}'.format(avg_reward)) # print the average reward value - print() - print('time:{0}s'.format(self.total_time)) # print the total time in seconds - return # stop training and return - self.loss=loss # update loss as the last loss value - self.loss_list.append(loss) # append loss to the loss list - i+=1 # increase the episode index by one - self.total_episode+=1 # increase the total episode counter by one - if episode_count%10!=0: - p=episode_count-episode_count%self.p - p=int(p/self.p) - s=episode_count-episode_count%self.s - s=int(s/self.s) - else: - p=episode_count/(self.p+1) - p=int(p) - s=episode_count/(self.s+1) - s=int(s) - if p==0: - p=1 - if s==0: - s=1 - if i%p==0: # check if i is a multiple of p or not - if len(self.state_pool)>=self.batch: # check if the pool size is larger than or equal to the batch size or not - print('episode:{0} loss:{1:.6f}'.format(i+1,loss)) # print the episode number and loss value - if avg_reward!=None: # check if avg_reward is None or not - print('episode:{0} average reward:{1}'.format(i+1,avg_reward)) # print the episode number and average reward valueelse: - else: - print('episode:{0} reward:{1}'.format(i+1,self.reward)) # print the episode number and reward value - print() - if save!=None and i%s==0: # check if save is None or not and if i is a multiple of s or not - self.save(self.total_episode,one) # save the network parameters and training data using the custom save function and the total episode number and one flag - if self.save_episode==True: # check if save episode flag is True or not - if done: # check if done flag is True or not - episode.append('done') # append 'done' to the episode data list - self.episode_set.append(episode) # append the episode data list to the episode set list - if self.max_episode_count!=None and len(self.episode_set)>=self.max_episode_count: # check if max episode count is defined or not and if the episode set size reaches the max episode count or not - self.save_episode=False # set the save episode flag as False to stop saving episodes - try: - try: - self.nn.ec.assign_add(1) # increase the episode counter by one using TensorFlow assign_add method - except Exception: - self.nn.ec+=1 # increase the episode counter by one using Python addition operator - except Exception: - pass - t2=time.time() # record the end time of the episode - self.time+=(t2-t1) # accumulate the time over episodes - self._time=self.time-int(self.time) # get the decimal part of the time - if self._time<0.5: - self.total_time=int(self.time) # round down the time if the decimal part is less than 0.5 - else: - self.total_time=int(self.time)+1 # round up the time if the decimal part is greater than or equal to 0.5 - self.total_time+=self.time # add the time to the total time - print('last loss:{0:.6f}'.format(loss)) # print the last loss value - print('last reward:{0}'.format(self.reward)) # print the last reward value - print() - print('time:{0}s'.format(self.time)) # print the time in seconds - return # return - - - def visualize_reward(self): - print() - plt.figure(1) # create a figure object with number 1 - plt.plot(np.arange(self.total_episode),self.reward_list) # plot the reward list against the episode number - plt.xlabel('episode') # set the x-axis label as 'episode' - plt.ylabel('reward') # set the y-axis label as 'reward' - print('reward:{0:.6f}'.format(self.reward_list[-1])) # print the last reward value - return - - - def visualize_train(self): - print() - plt.figure(1) # create a figure object with number 1 - plt.plot(np.arange(self.total_episode),self.loss_list) # plot the loss list against the episode number - plt.title('train loss') # set the title of the figure as 'train loss' - plt.xlabel('episode') # set the x-axis label as 'episode' - plt.ylabel('loss') # set the y-axis label as 'loss' - print('loss:{0:.6f}'.format(self.loss_list[-1])) # print the last loss value - return diff --git a/Note 7.0 pv documentation/compiler/nc.txt b/Note 7.0 pv documentation/compiler/nc.txt deleted file mode 100644 index 14ea9e06..00000000 --- a/Note 7.0 pv documentation/compiler/nc.txt +++ /dev/null @@ -1,33 +0,0 @@ -Note Compiler - -Note Compiler can compile filename.n into filename.py. -Using Note compiler can use different grammar to write your neural network in filename.n. - -Grammar transition by Note Compiler: -Operator: -(object.*object)->tf.matmul(object1,object2) -(object.^)->tf.reverse(object) -(object1.|object2)->tf.concat([object1,object2],0) -(object1.||object2)->tf.concat([object1,object2],1) -(object./)->tf.math.sqrt(object) -(object1.=object2)->state_ops.assign(object1,object2) - -Init: -example: -param=zint32. [3,4]->param=tf.zeros([3,4],dtype=tf.int32) -return zint32. [3,4]->return tf.zeros([3,4],dtype=tf.int32) -param=uint32. (0,1)[3,4]->param=tf.random.uniform([3,4],0,1,dtype=tf.int32) -return uint32. (0,1)[3,4]->return tf.random.uniform([3,4],0,1,dtype=tf.int32) -param=nint32. (0,1)[3,4]->param=tf.random.normal([3,4],0,1,dtype=tf.int32) -return nint32. (0,1)[3,4]->return tf.random.normal([3,4],0,1,dtype=tf.int32) -param=oint32. [3,4]->param=tf.ones([3,4],dtype=tf.int32) -return oint32. [3,4]->return tf.ones([3,4],dtype=tf.int32) - -Using example: -import Note.create.nc as nc -c=nc.compiler('nn.n') -c.Compile() -#then you will get filename.py. - -Macro: -define. macro name string \ No newline at end of file diff --git a/Note 7.0 pv documentation/compiler/nn.n b/Note 7.0 pv documentation/compiler/nn.n deleted file mode 100644 index fdf619f1..00000000 --- a/Note 7.0 pv documentation/compiler/nn.n +++ /dev/null @@ -1,26 +0,0 @@ -import tensorflow as tf -define. Pi 3.14 - -class nn: #a simple example,a neural network class - def __init__(self): - self.weight1=tf.Variable(nfloat32. [784,64]) - self.bias1=tf.Variable(nfloat32. [64]) - self.weight2=tf.Variable(nfloat32. [64,64]) - self.bias2=tf.Variable(nfloat32. [64]) - self.weight3=tf.Variable(nfloat32. [64,10]) - self.bias3=tf.Variable(nfloat32. [10]) - self.param=[self.weight1,self.weight2,self.weight3,self.bias1,self.bias2,self.bias3] - self.opt=tf.keras.optimizers.Adam() - self.info='example' - - - def fp(self,data): - with tf.device('GPU:0'): - layer1=tf.nn.relu((data.*self.weight1)+self.bias1) - layer2=tf.nn.relu((layer1.*self.weight2)+self.bias2) - output=(layer2.*self.weight3)+self.bias3+Pi - return output - - - def loss(self,output,labels): - return tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=output,labels=labels)) \ No newline at end of file diff --git a/Note 7.0 pv documentation/compiler/nn.py b/Note 7.0 pv documentation/compiler/nn.py deleted file mode 100644 index 7d02bda4..00000000 --- a/Note 7.0 pv documentation/compiler/nn.py +++ /dev/null @@ -1,25 +0,0 @@ -import tensorflow as tf - -class nn: #a simple example,a neural network class - def __init__(self): - self.weight1=tf.Variable(tf.random.normal([784,64],dtype=tf.float32)) - self.bias1=tf.Variable(tf.random.normal([64],dtype=tf.float32)) - self.weight2=tf.Variable(tf.random.normal([64,64],dtype=tf.float32)) - self.bias2=tf.Variable(tf.random.normal([64],dtype=tf.float32)) - self.weight3=tf.Variable(tf.random.normal([64,10],dtype=tf.float32)) - self.bias3=tf.Variable(tf.random.normal([10],dtype=tf.float32)) - self.param=[self.weight1,self.weight2,self.weight3,self.bias1,self.bias2,self.bias3] - self.opt=tf.keras.optimizers.Adam() - self.info='example' - - - def fp(self,data): - with tf.device('GPU:0'): - layer1=tf.nn.relu(tf.matmul(data,self.weight1)+self.bias1) - layer2=tf.nn.relu(tf.matmul(layer1,self.weight2)+self.bias2) - output=tf.matmul(layer2,self.weight3)+self.bias3+3.14 - return output - - - def loss(self,output,labels): - return tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=output,labels=labels)) \ No newline at end of file From 1bfc2a870af8ddddf8cfd95a114a0ad660d95c94 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Fri, 7 Jul 2023 13:35:46 +0800 Subject: [PATCH 036/337] Add files via upload --- .../DL/neural network/pytorch/nn.py | 53 ++ .../DL/neural network/tensorflow/bert.py | 167 ++++ .../DL/neural network/tensorflow/cnn.py | 27 + .../neural network/tensorflow/layer/LSTM.py | 37 + .../tensorflow/layer/attn_LSTM.py | 44 ++ .../DL/neural network/tensorflow/layer/cnn.py | 50 ++ .../tensorflow/layer/cnn_lmix.py | 58 ++ .../DL/neural network/tensorflow/layer/nn.py | 27 + .../neural network/tensorflow/layer/nn_acc.py | 32 + .../tensorflow/layer/nn_clipping.py | 40 + .../tensorflow/layer/self_LSTM.py | 45 ++ .../tensorflow/layer/transformer.py | 46 ++ .../DL/neural network/tensorflow/lstm.py | 26 + .../DL/neural network/tensorflow/nn.py | 23 + .../DL/neural network/tensorflow/nn_acc.py | 28 + .../DL/neural network/tensorflow/nn_ol.py | 37 + .../neural network/tensorflow/process/LSTM.py | 46 ++ .../tensorflow/process/attn_LSTM.py | 53 ++ .../neural network/tensorflow/process/cnn.py | 59 ++ .../tensorflow/process/cnn_lmix.py | 67 ++ .../neural network/tensorflow/process/nn.py | 39 + .../neural network/tensorflow/process/nn_.py | 39 + .../tensorflow/process/nn_acc.py | 44 ++ .../tensorflow/process/nn_attenuate.py | 49 ++ .../tensorflow/process/nn_clipping.py | 47 ++ .../tensorflow/process/self_LSTM.py | 54 ++ .../tensorflow/process/transformer.py | 55 ++ .../DL/reduced kernel/kernel_reduced.py | 741 ++++++++++++++++++ .../reduced kernel/process/kernel_reduced.py | 637 +++++++++++++++ Note 7.0 documentation/RL/kernel.txt | 10 + .../RL/neural network/pytorch/DDPG.py | 101 +++ .../RL/neural network/pytorch/DQN.py | 64 ++ .../RL/neural network/pytorch/DQN_pr.py | 76 ++ .../RL/neural network/pytorch/DoubleDQN.py | 65 ++ .../RL/neural network/pytorch/DuelingDQN.py | 67 ++ .../RL/neural network/tensorflow/DDPG.py | 84 ++ .../RL/neural network/tensorflow/DQN.py | 46 ++ .../RL/neural network/tensorflow/DQN_pr.py | 59 ++ .../neural network/tensorflow/pool net/DQN.py | 52 ++ .../tensorflow/pool net/thread/DDPG.py | 84 ++ .../tensorflow/pool net/thread/DQN.py | 46 ++ .../tensorflow/pool net/thread/DQN_m.py | 48 ++ .../tensorflow/pool net/thread/DQN_mtpr.py | 59 ++ .../RL/reduced kernel/kernel_reduced.py | 670 ++++++++++++++++ .../RL/reduced kernel/nspn/kernel_reduced.py | 685 ++++++++++++++++ Note 7.0 documentation/compiler/nc.txt | 33 + Note 7.0 documentation/compiler/nn.n | 26 + Note 7.0 documentation/compiler/nn.py | 25 + 48 files changed, 4970 insertions(+) create mode 100644 Note 7.0 documentation/DL/neural network/pytorch/nn.py create mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/bert.py create mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/cnn.py create mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/layer/LSTM.py create mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/layer/attn_LSTM.py create mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/layer/cnn.py create mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/layer/cnn_lmix.py create mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/layer/nn.py create mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/layer/nn_acc.py create mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/layer/nn_clipping.py create mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/layer/self_LSTM.py create mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/layer/transformer.py create mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/lstm.py create mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/nn.py create mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/nn_acc.py create mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/nn_ol.py create mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/process/LSTM.py create mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/process/attn_LSTM.py create mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/process/cnn.py create mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/process/cnn_lmix.py create mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/process/nn.py create mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/process/nn_.py create mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/process/nn_acc.py create mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/process/nn_attenuate.py create mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/process/nn_clipping.py create mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/process/self_LSTM.py create mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/process/transformer.py create mode 100644 Note 7.0 documentation/DL/reduced kernel/kernel_reduced.py create mode 100644 Note 7.0 documentation/DL/reduced kernel/process/kernel_reduced.py create mode 100644 Note 7.0 documentation/RL/kernel.txt create mode 100644 Note 7.0 documentation/RL/neural network/pytorch/DDPG.py create mode 100644 Note 7.0 documentation/RL/neural network/pytorch/DQN.py create mode 100644 Note 7.0 documentation/RL/neural network/pytorch/DQN_pr.py create mode 100644 Note 7.0 documentation/RL/neural network/pytorch/DoubleDQN.py create mode 100644 Note 7.0 documentation/RL/neural network/pytorch/DuelingDQN.py create mode 100644 Note 7.0 documentation/RL/neural network/tensorflow/DDPG.py create mode 100644 Note 7.0 documentation/RL/neural network/tensorflow/DQN.py create mode 100644 Note 7.0 documentation/RL/neural network/tensorflow/DQN_pr.py create mode 100644 Note 7.0 documentation/RL/neural network/tensorflow/pool net/DQN.py create mode 100644 Note 7.0 documentation/RL/neural network/tensorflow/pool net/thread/DDPG.py create mode 100644 Note 7.0 documentation/RL/neural network/tensorflow/pool net/thread/DQN.py create mode 100644 Note 7.0 documentation/RL/neural network/tensorflow/pool net/thread/DQN_m.py create mode 100644 Note 7.0 documentation/RL/neural network/tensorflow/pool net/thread/DQN_mtpr.py create mode 100644 Note 7.0 documentation/RL/reduced kernel/kernel_reduced.py create mode 100644 Note 7.0 documentation/RL/reduced kernel/nspn/kernel_reduced.py create mode 100644 Note 7.0 documentation/compiler/nc.txt create mode 100644 Note 7.0 documentation/compiler/nn.n create mode 100644 Note 7.0 documentation/compiler/nn.py diff --git a/Note 7.0 documentation/DL/neural network/pytorch/nn.py b/Note 7.0 documentation/DL/neural network/pytorch/nn.py new file mode 100644 index 00000000..1faec0fc --- /dev/null +++ b/Note 7.0 documentation/DL/neural network/pytorch/nn.py @@ -0,0 +1,53 @@ +import torch +from torch import nn + + +class NeuralNetwork(nn.Module): + def __init__(self): + super().__init__() + self.flatten=nn.Flatten() + self.linear_relu_stack=nn.Sequential( + nn.Linear(28*28, 512), + nn.ReLU(), + nn.Linear(512, 512), + nn.ReLU(), + nn.Linear(512, 10) + ) + + + def forward(self,x): + x=self.flatten(x) + logits=self.linear_relu_stack(x) + return logits + + +class neuralnetwork: + def __init__(self): + if torch.cuda.is_available(): + self.device=torch.device('cuda') + else: + self.device=torch.device('cpu') + self.model=NeuralNetwork().to(self.device) + self.loss_fn=nn.CrossEntropyLoss() + self.opt=torch.optim.SGD(self.model.parameters(),lr=1e-3) #optimizer,kernel uses it to optimize. + + + def fp(self,x): #forward propagation function,kernel uses it for forward propagation. + pred=self.model(x.to(self.device)) + return pred + + + def loss(self,output,labels): #loss functino,kernel uses it to calculate loss. + loss=self.loss_fn(output,labels.to(self.device)) + return loss + + + def backward(self,loss): #backward function,kernel uses it for backpropagation. + self.optim.zero_grad() + loss.backward() + return + + + def opt(self): #opt function,kernel uses it to optimize. + self.optim.step() + return \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/bert.py b/Note 7.0 documentation/DL/neural network/tensorflow/bert.py new file mode 100644 index 00000000..9715477f --- /dev/null +++ b/Note 7.0 documentation/DL/neural network/tensorflow/bert.py @@ -0,0 +1,167 @@ +import tensorflow as tf +import tensorflow_hub as hub + + +class bert: + def __init__(self): + text_input=tf.keras.layers.Input(shape=(),dtype=tf.string,name='text') + preprocessing_layer=hub.KerasLayer(tfhub_handle_preprocess,name='preprocessing') + encoder_inputs=preprocessing_layer(text_input) + encoder=hub.KerasLayer(tfhub_handle_encoder,trainable=True,name='BERT_encoder') + outputs=encoder(encoder_inputs) + net=outputs['pooled_output'] + net=tf.keras.layers.Dropout(0.1)(net) + net=tf.keras.layers.Dense(1,activation=None,name='classifier')(net) + self.model=tf.keras.Model(text_input,net) + self.param=self.model.weights + self.opt=tf.keras.optimizers.Adam() + + + def fp(self,data): + return self.model(data) + + + def loss(self,output,labels): + bce=tf.keras.losses.BinaryCrossentropy(from_logits=True) + return bce(output,labels) + + +bert_model_name = 'small_bert/bert_en_uncased_L-4_H-512_A-8' +map_name_to_handle = { + 'bert_en_uncased_L-12_H-768_A-12': + 'https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/3', + 'bert_en_cased_L-12_H-768_A-12': + 'https://tfhub.dev/tensorflow/bert_en_cased_L-12_H-768_A-12/3', + 'bert_multi_cased_L-12_H-768_A-12': + 'https://tfhub.dev/tensorflow/bert_multi_cased_L-12_H-768_A-12/3', + 'small_bert/bert_en_uncased_L-2_H-128_A-2': + 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-2_H-128_A-2/1', + 'small_bert/bert_en_uncased_L-2_H-256_A-4': + 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-2_H-256_A-4/1', + 'small_bert/bert_en_uncased_L-2_H-512_A-8': + 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-2_H-512_A-8/1', + 'small_bert/bert_en_uncased_L-2_H-768_A-12': + 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-2_H-768_A-12/1', + 'small_bert/bert_en_uncased_L-4_H-128_A-2': + 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-128_A-2/1', + 'small_bert/bert_en_uncased_L-4_H-256_A-4': + 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-256_A-4/1', + 'small_bert/bert_en_uncased_L-4_H-512_A-8': + 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-512_A-8/1', + 'small_bert/bert_en_uncased_L-4_H-768_A-12': + 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-768_A-12/1', + 'small_bert/bert_en_uncased_L-6_H-128_A-2': + 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-6_H-128_A-2/1', + 'small_bert/bert_en_uncased_L-6_H-256_A-4': + 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-6_H-256_A-4/1', + 'small_bert/bert_en_uncased_L-6_H-512_A-8': + 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-6_H-512_A-8/1', + 'small_bert/bert_en_uncased_L-6_H-768_A-12': + 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-6_H-768_A-12/1', + 'small_bert/bert_en_uncased_L-8_H-128_A-2': + 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-8_H-128_A-2/1', + 'small_bert/bert_en_uncased_L-8_H-256_A-4': + 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-8_H-256_A-4/1', + 'small_bert/bert_en_uncased_L-8_H-512_A-8': + 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-8_H-512_A-8/1', + 'small_bert/bert_en_uncased_L-8_H-768_A-12': + 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-8_H-768_A-12/1', + 'small_bert/bert_en_uncased_L-10_H-128_A-2': + 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-10_H-128_A-2/1', + 'small_bert/bert_en_uncased_L-10_H-256_A-4': + 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-10_H-256_A-4/1', + 'small_bert/bert_en_uncased_L-10_H-512_A-8': + 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-10_H-512_A-8/1', + 'small_bert/bert_en_uncased_L-10_H-768_A-12': + 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-10_H-768_A-12/1', + 'small_bert/bert_en_uncased_L-12_H-128_A-2': + 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-12_H-128_A-2/1', + 'small_bert/bert_en_uncased_L-12_H-256_A-4': + 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-12_H-256_A-4/1', + 'small_bert/bert_en_uncased_L-12_H-512_A-8': + 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-12_H-512_A-8/1', + 'small_bert/bert_en_uncased_L-12_H-768_A-12': + 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-12_H-768_A-12/1', + 'albert_en_base': + 'https://tfhub.dev/tensorflow/albert_en_base/2', + 'electra_small': + 'https://tfhub.dev/google/electra_small/2', + 'electra_base': + 'https://tfhub.dev/google/electra_base/2', + 'experts_pubmed': + 'https://tfhub.dev/google/experts/bert/pubmed/2', + 'experts_wiki_books': + 'https://tfhub.dev/google/experts/bert/wiki_books/2', + 'talking-heads_base': + 'https://tfhub.dev/tensorflow/talkheads_ggelu_bert_en_base/1', +} +map_model_to_preprocess = { + 'bert_en_uncased_L-12_H-768_A-12': + 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', + 'bert_en_cased_L-12_H-768_A-12': + 'https://tfhub.dev/tensorflow/bert_en_cased_preprocess/3', + 'small_bert/bert_en_uncased_L-2_H-128_A-2': + 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', + 'small_bert/bert_en_uncased_L-2_H-256_A-4': + 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', + 'small_bert/bert_en_uncased_L-2_H-512_A-8': + 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', + 'small_bert/bert_en_uncased_L-2_H-768_A-12': + 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', + 'small_bert/bert_en_uncased_L-4_H-128_A-2': + 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', + 'small_bert/bert_en_uncased_L-4_H-256_A-4': + 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', + 'small_bert/bert_en_uncased_L-4_H-512_A-8': + 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', + 'small_bert/bert_en_uncased_L-4_H-768_A-12': + 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', + 'small_bert/bert_en_uncased_L-6_H-128_A-2': + 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', + 'small_bert/bert_en_uncased_L-6_H-256_A-4': + 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', + 'small_bert/bert_en_uncased_L-6_H-512_A-8': + 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', + 'small_bert/bert_en_uncased_L-6_H-768_A-12': + 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', + 'small_bert/bert_en_uncased_L-8_H-128_A-2': + 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', + 'small_bert/bert_en_uncased_L-8_H-256_A-4': + 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', + 'small_bert/bert_en_uncased_L-8_H-512_A-8': + 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', + 'small_bert/bert_en_uncased_L-8_H-768_A-12': + 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', + 'small_bert/bert_en_uncased_L-10_H-128_A-2': + 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', + 'small_bert/bert_en_uncased_L-10_H-256_A-4': + 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', + 'small_bert/bert_en_uncased_L-10_H-512_A-8': + 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', + 'small_bert/bert_en_uncased_L-10_H-768_A-12': + 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', + 'small_bert/bert_en_uncased_L-12_H-128_A-2': + 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', + 'small_bert/bert_en_uncased_L-12_H-256_A-4': + 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', + 'small_bert/bert_en_uncased_L-12_H-512_A-8': + 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', + 'small_bert/bert_en_uncased_L-12_H-768_A-12': + 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', + 'bert_multi_cased_L-12_H-768_A-12': + 'https://tfhub.dev/tensorflow/bert_multi_cased_preprocess/3', + 'albert_en_base': + 'https://tfhub.dev/tensorflow/albert_en_preprocess/3', + 'electra_small': + 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', + 'electra_base': + 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', + 'experts_pubmed': + 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', + 'experts_wiki_books': + 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', + 'talking-heads_base': + 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', +} +tfhub_handle_encoder=map_name_to_handle[bert_model_name] +tfhub_handle_preprocess=map_model_to_preprocess[bert_model_name] \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/cnn.py b/Note 7.0 documentation/DL/neural network/tensorflow/cnn.py new file mode 100644 index 00000000..bd7ad1bf --- /dev/null +++ b/Note 7.0 documentation/DL/neural network/tensorflow/cnn.py @@ -0,0 +1,27 @@ +import tensorflow as tf +from tensorflow.keras import layers,models + + +class cnn: + def __init__(self): + self.model=models.Sequential() + self.model.add(layers.Conv2D(32,(3,3),activation='relu',input_shape=(32,32,3))) + self.model.add(layers.MaxPooling2D((2,2))) + self.model.add(layers.Conv2D(64,(3,3),activation='relu')) + self.model.add(layers.MaxPooling2D((2,2))) + self.model.add(layers.Conv2D(64,(3,3),activation='relu')) + self.model.add(layers.Flatten()) + self.model.add(layers.Dense(64,activation='relu')) + self.model.add(layers.Dense(10)) + self.param=self.model.weights + self.opt=tf.keras.optimizers.Adam() + + + def fp(self,data): + output=self.model(data) + return output + + + def loss(self,output,labels): + loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + return loss(labels,output) \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/layer/LSTM.py b/Note 7.0 documentation/DL/neural network/tensorflow/layer/LSTM.py new file mode 100644 index 00000000..272855a3 --- /dev/null +++ b/Note 7.0 documentation/DL/neural network/tensorflow/layer/LSTM.py @@ -0,0 +1,37 @@ +import tensorflow as tf +from Note.nn.layer.LSTM import LSTM +from Note.nn.layer.dense import dense + +# Define a recurrent neural network class with LSTM layers +class lstm: + def __init__(self): + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.opt=tf.keras.optimizers.Adam() + + + def build(self): + # Create two LSTM layers with 50 hidden units each + self.lstm1=LSTM(weight_shape=(3,50),return_sequence=True) # First LSTM layer that returns the full sequence + self.lstm2=LSTM(weight_shape=(50,50),) # Second LSTM layer that returns the last output only + # Create a dense layer with 10 output units + self.dense=dense([50, 10]) + # Store the parameters of the layers in a list + self.param=[self.lstm1.param, + self.lstm2.param, + self.dense.param, + ] + return + + + def fp(self,data): + # Perform forward propagation on the input data + x=self.lstm1.output(data) # First LSTM layer output + x=self.lstm2.output(x) # Second LSTM layer output + output=self.dense.output(x) # Dense layer output + return output + + + def loss(self, output, labels): + # Compute the loss between the output and the labels + return self.loss_object(labels, output) \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/layer/attn_LSTM.py b/Note 7.0 documentation/DL/neural network/tensorflow/layer/attn_LSTM.py new file mode 100644 index 00000000..1d41befd --- /dev/null +++ b/Note 7.0 documentation/DL/neural network/tensorflow/layer/attn_LSTM.py @@ -0,0 +1,44 @@ +import tensorflow as tf +from Note.nn.layer.LSTM import LSTM +from Note.nn.layer.dense import dense +from Note.nn.layer.attention import attention + +# Define a recurrent neural network class with LSTM layers and attention +class attn_lstm: + def __init__(self): + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.opt=tf.keras.optimizers.Adam() + + + def build(self): + # Create two LSTM layers with 50 hidden units each + self.lstm1=LSTM(weight_shape=(3,50),return_sequence=True) # First LSTM layer that returns the full sequence + self.lstm2=LSTM(weight_shape=(50,50),) # Second LSTM layer that returns the last output only + # Create a dense layer with 10 output units and softmax activation + self.dense=dense([100, 10],activation='softmax') + # Create an attention object with weight shape (50, 50) + self.attn=attention((50,50)) + # Store the parameters of the layers and the attention in a list + self.param=[self.lstm1.param, + self.lstm2.param, + self.dense.param, + self.attn.param] # Add the attention parameters to the list + return + + + def fp(self,data): + # Perform forward propagation on the input data + x=self.lstm1.output(data) # First LSTM layer output, shape (B, N, 50) + y=self.lstm2.output(x) # Second LSTM layer output, shape (B, 50) + # Use the attention object to compute the context vector from x and y, shape (B, 50) + context_vector,_,_=self.attn.output(x,y) + # Concatenate the context vector and the second LSTM output, shape (B, 100) + z=tf.concat([context_vector,y],axis=-1) + output=self.dense.output(z) # Dense layer output with softmax activation, shape (B, 10) + return output + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/layer/cnn.py b/Note 7.0 documentation/DL/neural network/tensorflow/layer/cnn.py new file mode 100644 index 00000000..aa2667f8 --- /dev/null +++ b/Note 7.0 documentation/DL/neural network/tensorflow/layer/cnn.py @@ -0,0 +1,50 @@ +import tensorflow as tf +from Note.nn.layer.conv2d import conv2d +from Note.nn.layer.dense import dense +from Note.nn.layer.flatten import flatten + +# Define a convolutional neural network class +class cnn: + def __init__(self): + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.opt=tf.keras.optimizers.Adam() + + + def build(self): + # Create three convolutional layers with relu activations + self.conv1=conv2d(weight_shape=(3,3,3,32),activation='relu') + self.conv2=conv2d(weight_shape=(3,3,32,64),activation='relu') + self.conv3=conv2d(weight_shape=(3,3,64,64),activation='relu') + # Create two dense layers with relu and linear activations + self.dense1=dense([64*4*4,64],activation='relu') + self.dense2=dense([64,10]) + # Store the parameters of the layers in a list + self.param = [self.conv1.weight, + self.conv2.weight, + self.conv3.weight, + self.dense1.weight, + self.dense1.bias, + self.dense2.weight, + self.dense2.bias] + return + + + def fp(self,data): + # Perform forward propagation on the input data + x=tf.nn.conv2d(data,self.conv1.weight,strides=(1,1),padding='SAME') # First convolutional layer + x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # First max pooling layer + x=tf.nn.conv2d(x,self.conv2.weight,strides=(1,1),padding='SAME') # Second convolutional layer + x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # Second max pooling layer + x=tf.nn.conv2d(x,self.conv3.weight,strides=(1,1),padding='SAME') # Third convolutional layer + x=flatten(x) # Flatten the output to a vector + x=tf.nn.relu(tf.matmul(x,self.dense1.weight)+self.dense1.bias) # First dense layer with relu activation + x=tf.nn.dropout(x,rate=0.5) # Apply dropout to prevent overfitting + output=tf.matmul(x,self.dense2.weight)+self.dense2.bias # Output layer with linear activation + output=tf.nn.dropout(output,rate=0.5) # Apply dropout to prevent overfitting + return output + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/layer/cnn_lmix.py b/Note 7.0 documentation/DL/neural network/tensorflow/layer/cnn_lmix.py new file mode 100644 index 00000000..40232919 --- /dev/null +++ b/Note 7.0 documentation/DL/neural network/tensorflow/layer/cnn_lmix.py @@ -0,0 +1,58 @@ +import tensorflow as tf +from Note.nn.layer.conv2d import conv2d +from Note.nn.layer.dense import dense +from Note.nn.layer.flatten import flatten +from Note.nn.layer.LMix import lmix + +# Define a convolutional neural network class with mixup augmentation +class cnn: + def __init__(self): + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.opt=tf.keras.optimizers.Adam() + # Initialize the alpha parameter for mixup + self.alpha=1.0 + + + def build(self): + # Create three convolutional layers with relu activations + self.conv1=conv2d(weight_shape=(3,3,3,32),activation='relu') + self.conv2=conv2d(weight_shape=(3,3,32,64),activation='relu') + self.conv3=conv2d(weight_shape=(3,3,64,64),activation='relu') + # Create two dense layers with relu and linear activations + self.dense1=dense([64*4*4,64],activation='relu') + self.dense2=dense([64,10]) + # Store the parameters of the layers in a list + self.param = [self.conv1.weight, + self.conv2.weight, + self.conv3.weight, + self.dense1.weight, + self.dense1.bias, + self.dense2.weight, + self.dense2.bias] + return + + + def data_func(self,data_batch,labels_batch): + # Apply mixup augmentation on the input data and labels + return lmix(data_batch,labels_batch,self.alpha) + + + def fp(self,data): + # Perform forward propagation on the input data + x=tf.nn.conv2d(data,self.conv1.weight,strides=(1,1),padding='SAME') # First convolutional layer + x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # First max pooling layer + x=tf.nn.conv2d(x,self.conv2.weight,strides=(1,1),padding='SAME') # Second convolutional layer + x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # Second max pooling layer + x=tf.nn.conv2d(x,self.conv3.weight,strides=(1,1),padding='SAME') # Third convolutional layer + x=flatten(x) # Flatten the output to a vector + x=tf.nn.relu(tf.matmul(x,self.dense1.weight)+self.dense1.bias) # First dense layer with relu activation + x=tf.nn.dropout(x,rate=0.5) # Apply dropout to prevent overfitting + output=tf.matmul(x,self.dense2.weight)+self.dense2.bias # Output layer with linear activation + output=tf.nn.dropout(output,rate=0.5) # Apply dropout to prevent overfitting + return output + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/layer/nn.py b/Note 7.0 documentation/DL/neural network/tensorflow/layer/nn.py new file mode 100644 index 00000000..77ed7454 --- /dev/null +++ b/Note 7.0 documentation/DL/neural network/tensorflow/layer/nn.py @@ -0,0 +1,27 @@ +import tensorflow as tf +import Note.nn.layer.dense as d +from Note.nn.layer.flatten import flatten + + +class nn: + def __init__(self): + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.opt=tf.keras.optimizers.Adam() + + + def build(self): + self.layer1=d.dense([784,128],activation='relu') + self.layer2=d.dense([128,10]) + self.param=[self.layer1.param,self.layer2.param] + return + + + def fp(self,data): + data=flatten(data) + output1=self.layer1.output(data) + output2=self.layer2.output(output1) + return output2 + + + def loss(self,output,labels): + return self.loss_object(labels,output) \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/layer/nn_acc.py b/Note 7.0 documentation/DL/neural network/tensorflow/layer/nn_acc.py new file mode 100644 index 00000000..cad20961 --- /dev/null +++ b/Note 7.0 documentation/DL/neural network/tensorflow/layer/nn_acc.py @@ -0,0 +1,32 @@ +import tensorflow as tf +import Note.nn.layer.dense as d +from Note.nn.layer.flatten import flatten + + +class nn: + def __init__(self): + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.train_accuracy=tf.keras.metrics.SparseCategoricalAccuracy(name='train_accuracy') + self.opt=tf.keras.optimizers.Adam() + + + def build(self): + self.layer1=d.dense([784,128],activation='relu') + self.layer2=d.dense([128,10]) + self.param=[self.layer1.param,self.layer2.param] + return + + + def fp(self,data): + data=flatten(data) + output1=self.layer1.output(data) + output2=self.layer2.output(output1) + return output2 + + + def loss(self,output,labels): + return self.loss_object(labels,output) + + + def accuracy(self,output,labels): + return self.train_accuracy(labels,output) \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/layer/nn_clipping.py b/Note 7.0 documentation/DL/neural network/tensorflow/layer/nn_clipping.py new file mode 100644 index 00000000..b53a43da --- /dev/null +++ b/Note 7.0 documentation/DL/neural network/tensorflow/layer/nn_clipping.py @@ -0,0 +1,40 @@ +import tensorflow as tf +import Note.nn.layer.dense as d +from Note.nn.layer.flatten import flatten +from Note.nn.gradient_clipping import gradient_clipping + +# Define a neural network class +class nn: + def __init__(self): + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.opt=tf.keras.optimizers.Adam() + + + def build(self): + # Create two dense layers with relu and linear activations + self.layer1=d.dense([784,128],activation='relu') + self.layer2=d.dense([128,10]) + # Store the parameters of the layers in a list + self.param=[self.layer1.param,self.layer2.param] + return + + + def fp(self,data): + # Perform forward propagation on the input data + data=flatten(data) + output1=self.layer1.output(data) + output2=self.layer2.output(output1) + return output2 + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) + + + def gradient(self,tape,loss): + # Compute the gradients of the loss with respect to the parameters + grads=tape.gradient(loss,self.param) + # Clip the gradients by value to avoid exploding gradients + return gradient_clipping(grads,'value',1.0) \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/layer/self_LSTM.py b/Note 7.0 documentation/DL/neural network/tensorflow/layer/self_LSTM.py new file mode 100644 index 00000000..edaa2420 --- /dev/null +++ b/Note 7.0 documentation/DL/neural network/tensorflow/layer/self_LSTM.py @@ -0,0 +1,45 @@ +import tensorflow as tf +from Note.nn.layer.LSTM import LSTM +from Note.nn.layer.self_attention import self_attention +from Note.nn.layer.dense import dense +from Note.nn.layer.emdedding import embedding + +# Define a LSTM self attention neural network class +class self_LSTM: + def __init__(self,vocab_size,embed_size,num_heads,num_layers): + self.vocab_size=vocab_size + self.embed_size=embed_size + self.num_heads=num_heads + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.opt=tf.keras.optimizers.Adam() + + + def build(self): + # Create an embedding layer to map input tokens to vectors + self.embedding_layer = embedding(self.vocab_size,self.embed_size) + # Create a LSTM layer to process the input sequence + self.lstm_layer = LSTM(weight_shape=[self.embed_size,self.embed_size],return_sequence=True) + # Create a self attention layer to attend to the LSTM output + self.self_attention_layer = self_attention(weight_shape=[self.embed_size,self.embed_size],num_heads=self.num_heads) + # Create a linear layer to map the attention output to logits + self.logits_layer = dense([self.embed_size,self.vocab_size]) + # Store the parameters of the layers in a list + self.param = [self.embedding_layer.embeddings] + self.param.extend(self.lstm_layer.param) + self.param.extend(self.self_attention_layer.param) + self.param.extend(self.logits_layer.param) + + + def fp(self,data): + # Perform forward propagation on the input data + x = self.embedding_layer.output(data) # shape: (batch_size, seq_len, embed_size) + x = self.lstm_layer.output(x) # shape: (batch_size, seq_len, embed_size) + x, attention_weights = self.self_attention_layer.output(x,self.num_heads) # shape: (batch_size, seq_len, embed_size) + output = self.logits_layer.output(x) # shape: (batch_size, seq_len, vocab_size) + return output + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/layer/transformer.py b/Note 7.0 documentation/DL/neural network/tensorflow/layer/transformer.py new file mode 100644 index 00000000..47862006 --- /dev/null +++ b/Note 7.0 documentation/DL/neural network/tensorflow/layer/transformer.py @@ -0,0 +1,46 @@ +import tensorflow as tf +from Note.nn.layer.transformer import transformer +from Note.nn.layer.dense import dense +from Note.nn.layer.emdedding import embedding +from Note.nn.positional_encoding import positional_encoding + +# Define a positional encoding transformer neural network class +class Transformer: + def __init__(self,vocab_size,embed_size,num_heads,num_layers,max_len): + self.vocab_size=vocab_size + self.embed_size=embed_size + self.num_heads=num_heads + self.num_layers=num_layers + self.max_len=max_len + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.opt=tf.keras.optimizers.Adam() + + + def build(self): + # Create an embedding layer to map input tokens to vectors + self.embedding_layer = embedding(self.vocab_size,self.embed_size) + # Create a list of transformer layers + self.transformer_layers = [transformer(weight_shape=[self.embed_size, self.embed_size], num_heads=self.num_heads) for _ in range(self.num_layers)] + # Create a linear layer to map the transformer output to logits + self.logits_layer = dense([self.embed_size,self.vocab_size]) + # Store the parameters of the layers in a list + self.param = [self.embedding_layer.embeddings] + for transformer_layer in self.transformer_layers: + self.param.extend(transformer_layer.param) + self.param.extend(self.logits_layer.param) + + + def fp(self,data): + # Perform forward propagation on the input data + x = self.embedding_layer.output(data) # shape: (batch_size, seq_len, embed_size) + x += positional_encoding(self.max_len,self.embed_size) # shape: (batch_size, seq_len, embed_size) + for transformer_layer in self.transformer_layers: + x = transformer_layer.output(x) # shape: (batch_size, seq_len, embed_size) + output = self.logits_layer.output(x) # shape: (batch_size, seq_len, vocab_size) + return output + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/lstm.py b/Note 7.0 documentation/DL/neural network/tensorflow/lstm.py new file mode 100644 index 00000000..29b9b396 --- /dev/null +++ b/Note 7.0 documentation/DL/neural network/tensorflow/lstm.py @@ -0,0 +1,26 @@ +import tensorflow as tf + + +class lstm: + def __init__(self,encoder): + self.model=tf.keras.Sequential([ + encoder, + tf.keras.layers.Embedding(len(encoder.get_vocabulary()), 64, mask_zero=True), + tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(64, return_sequences=True)), + tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(32)), + tf.keras.layers.Dense(64, activation='relu'), + tf.keras.layers.Dropout(0.5), + tf.keras.layers.Dense(1) + ]) + self.param=self.model.weights[1:] + self.opt=tf.keras.optimizers.Adam() + + + def fp(self,data): + output=self.model(data) + return output + + + def loss(self,output,labels): + bce=tf.keras.losses.BinaryCrossentropy(from_logits=True) + return bce(labels,output) \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/nn.py b/Note 7.0 documentation/DL/neural network/tensorflow/nn.py new file mode 100644 index 00000000..b4f03a92 --- /dev/null +++ b/Note 7.0 documentation/DL/neural network/tensorflow/nn.py @@ -0,0 +1,23 @@ +import tensorflow as tf + + +class nn: + def __init__(self): + self.model=tf.keras.models.Sequential([ + tf.keras.layers.Flatten(input_shape=(28, 28)), + tf.keras.layers.Dense(128,activation='relu'), + tf.keras.layers.Dropout(0.2), + tf.keras.layers.Dense(10) + ]) + self.param=self.model.weights #parameter list,kernel uses it list for backpropagation. + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.opt=tf.keras.optimizers.Adam() #optimizer,kernel uses it to optimize. + + + def fp(self,data): #forward propagation function,kernel uses it for forward propagation. + output=self.model(data) + return output + + + def loss(self,output,labels): #loss functino,kernel uses it to calculate loss. + return self.loss_object(labels,output) \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/nn_acc.py b/Note 7.0 documentation/DL/neural network/tensorflow/nn_acc.py new file mode 100644 index 00000000..0a726603 --- /dev/null +++ b/Note 7.0 documentation/DL/neural network/tensorflow/nn_acc.py @@ -0,0 +1,28 @@ +import tensorflow as tf + +#An example with accuracy function. +class nn: + def __init__(self): + self.model=tf.keras.models.Sequential([ + tf.keras.layers.Flatten(input_shape=(28, 28)), + tf.keras.layers.Dense(128,activation='relu'), + tf.keras.layers.Dropout(0.2), + tf.keras.layers.Dense(10) + ]) + self.param=self.model.weights #parameter list,kernel uses it list for backpropagation. + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.train_accuracy=tf.keras.metrics.SparseCategoricalAccuracy(name='train_accuracy') + self.opt=tf.keras.optimizers.Adam() #optimizer,kernel uses it to optimize. + + + def fp(self,data): #forward propagation function,kernel uses it for forward propagation. + output=self.model(data) + return output + + + def loss(self,output,labels): #loss functino,kernel uses it to calculate loss. + return self.loss_object(labels,output) + + + def accuracy(self,output,labels): #accuracy function,kernel uses it to calculate accuracy. + return self.train_accuracy(labels,output) \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/nn_ol.py b/Note 7.0 documentation/DL/neural network/tensorflow/nn_ol.py new file mode 100644 index 00000000..c32bc542 --- /dev/null +++ b/Note 7.0 documentation/DL/neural network/tensorflow/nn_ol.py @@ -0,0 +1,37 @@ +import tensorflow as tf +import numpy as np + +#This is an example of online training. +class nn: + def __init__(self,data,labels): + self.train_data=data + self.train_labels=labels + self.model=tf.keras.models.Sequential([ + tf.keras.layers.Flatten(input_shape=(28, 28)), + tf.keras.layers.Dense(128,activation='relu'), + tf.keras.layers.Dropout(0.2), + tf.keras.layers.Dense(10) + ]) + self.param=self.model.weights #parameter list,kernel uses it list for backpropagation. + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.opt=tf.keras.optimizers.Adam() #optimizer,kernel uses it to optimize. + self.train_loss_list=[] + self.c=0 #counter + self.max_length=1000 + + + def ol(self): #This is simulative online function. + index=np.random.choice(60000,size=[32]) + if self.c==10000: + return 'stop' + else: + return [self.train_data[index],self.train_labels[index]] + + + def fp(self,data): #forward propagation function,kernel uses it for forward propagation. + output=self.model(data) + return output + + + def loss(self,output,labels): #loss functino,kernel uses it to calculate loss. + return self.loss_object(labels,output) \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/process/LSTM.py b/Note 7.0 documentation/DL/neural network/tensorflow/process/LSTM.py new file mode 100644 index 00000000..93ac8bf0 --- /dev/null +++ b/Note 7.0 documentation/DL/neural network/tensorflow/process/LSTM.py @@ -0,0 +1,46 @@ +import tensorflow as tf +from Note.nn.layer.LSTM import LSTM +from Note.nn.layer.dense import dense +from Note.nn.process.optimizer import Adam + +# Define a recurrent neural network class with LSTM layers +class lstm: + def __init__(self): + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.optimizer=Adam() + # Initialize a variable to keep track of the batch count + self.bc=tf.Variable(0, dtype=tf.float32) + + + def build(self): + # Create two LSTM layers with 50 hidden units each + self.lstm1=LSTM(weight_shape=(3,50),return_sequence=True) # First LSTM layer that returns the full sequence + self.lstm2=LSTM(weight_shape=(50,50),) # Second LSTM layer that returns the last output only + # Create a dense layer with 10 output units + self.dense=dense([50, 10]) + # Store the parameters of the layers in a list + self.param=[self.lstm1.param, + self.lstm2.param, + self.dense.param, + ] + return + + + def fp(self,data): + # Perform forward propagation on the input data + x=self.lstm1.output(data) # First LSTM layer output + x=self.lstm2.output(x) # Second LSTM layer output + output=self.dense.output(x) # Dense layer output + return output + + + def loss(self, output, labels): + # Compute the loss between the output and the labels + return self.loss_object(labels, output) + + + def opt(self,gradient): + # Perform optimization on the parameters using the gradient and the batch count + param=self.optimizer.opt(gradient,self.param,self.bc[0]) + return param \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/process/attn_LSTM.py b/Note 7.0 documentation/DL/neural network/tensorflow/process/attn_LSTM.py new file mode 100644 index 00000000..2326ad57 --- /dev/null +++ b/Note 7.0 documentation/DL/neural network/tensorflow/process/attn_LSTM.py @@ -0,0 +1,53 @@ +import tensorflow as tf +from Note.nn.layer.LSTM import LSTM +from Note.nn.layer.dense import dense +from Note.nn.layer.attention import attention +from Note.nn.process.optimizer import Adam + +# Define a recurrent neural network class with LSTM layers and attention +class attn_lstm: + def __init__(self): + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.optimizer=Adam() + # Initialize a variable to keep track of the batch count + self.bc=tf.Variable(0, dtype=tf.float32) + + + def build(self): + # Create two LSTM layers with 50 hidden units each + self.lstm1=LSTM(weight_shape=(3,50),return_sequence=True) # First LSTM layer that returns the full sequence + self.lstm2=LSTM(weight_shape=(50,50),) # Second LSTM layer that returns the last output only + # Create a dense layer with 10 output units and softmax activation + self.dense=dense([100, 10],activation='softmax') + # Create an attention object with weight shape (50, 50) + self.attn=attention((50,50)) + # Store the parameters of the layers and the attention in a list + self.param=[self.lstm1.param, + self.lstm2.param, + self.dense.param, + self.attn.param] # Add the attention parameters to the list + return + + + def fp(self,data): + # Perform forward propagation on the input data + x=self.lstm1.output(data) # First LSTM layer output, shape (B, N, 50) + y=self.lstm2.output(x) # Second LSTM layer output, shape (B, 50) + # Use the attention object to compute the context vector from x and y, shape (B, 50) + context_vector,_,_=self.attn.output(x,y) + # Concatenate the context vector and the second LSTM output, shape (B, 100) + z=tf.concat([context_vector,y],axis=-1) + output=self.dense.output(z) # Dense layer output with softmax activation, shape (B, 10) + return output + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) + + + def opt(self,gradient): + # Perform optimization on the parameters using the gradient and the batch count + param=self.optimizer.opt(gradient,self.param,self.bc[0]) + return param \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/process/cnn.py b/Note 7.0 documentation/DL/neural network/tensorflow/process/cnn.py new file mode 100644 index 00000000..49892ccc --- /dev/null +++ b/Note 7.0 documentation/DL/neural network/tensorflow/process/cnn.py @@ -0,0 +1,59 @@ +import tensorflow as tf +from Note.nn.layer.conv2d import conv2d +from Note.nn.layer.dense import dense +from Note.nn.layer.flatten import flatten +from Note.nn.process.optimizer import Adam + +# Define a convolutional neural network class +class cnn: + def __init__(self): + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.optimizer=Adam() + # Initialize a variable to keep track of the batch count + self.bc=tf.Variable(0,dtype=tf.float32) + + + def build(self): + # Create three convolutional layers with relu activations + self.conv1=conv2d(weight_shape=(3,3,3,32),activation='relu') + self.conv2=conv2d(weight_shape=(3,3,32,64),activation='relu') + self.conv3=conv2d(weight_shape=(3,3,64,64),activation='relu') + # Create two dense layers with relu and linear activations + self.dense1=dense([64*4*4,64],activation='relu') + self.dense2=dense([64,10]) + # Store the parameters of the layers in a list + self.param = [self.conv1.weight, + self.conv2.weight, + self.conv3.weight, + self.dense1.weight, + self.dense1.bias, + self.dense2.weight, + self.dense2.bias] + return + + + def fp(self,data): + # Perform forward propagation on the input data + x=tf.nn.conv2d(data,self.conv1.weight,strides=(1,1),padding='SAME') # First convolutional layer + x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # First max pooling layer + x=tf.nn.conv2d(x,self.conv2.weight,strides=(1,1),padding='SAME') # Second convolutional layer + x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # Second max pooling layer + x=tf.nn.conv2d(x,self.conv3.weight,strides=(1,1),padding='SAME') # Third convolutional layer + x=flatten(x) # Flatten the output to a vector + x=tf.nn.relu(tf.matmul(x,self.dense1.weight)+self.dense1.bias) # First dense layer with relu activation + x=tf.nn.dropout(x,rate=0.5) # Apply dropout to prevent overfitting + output=tf.matmul(x,self.dense2.weight)+self.dense2.bias # Output layer with linear activation + output=tf.nn.dropout(output,rate=0.5) # Apply dropout to prevent overfitting + return output + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) + + + def opt(self,gradient): + # Perform optimization on the parameters using the gradient and the batch count + param=self.optimizer.opt(gradient,self.param,self.bc[0]) + return param \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/process/cnn_lmix.py b/Note 7.0 documentation/DL/neural network/tensorflow/process/cnn_lmix.py new file mode 100644 index 00000000..c1fa4cc9 --- /dev/null +++ b/Note 7.0 documentation/DL/neural network/tensorflow/process/cnn_lmix.py @@ -0,0 +1,67 @@ +import tensorflow as tf +from Note.nn.layer.conv2d import conv2d +from Note.nn.layer.dense import dense +from Note.nn.layer.flatten import flatten +from Note.nn.process.optimizer import Adam +from Note.nn.layer.LMix import lmix + +# Define a convolutional neural network class with mixup augmentation +class cnn: + def __init__(self): + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.optimizer=Adam() + # Initialize the alpha parameter for mixup + self.alpha=1.0 + # Initialize a variable to keep track of the batch count + self.bc=tf.Variable(0,dtype=tf.float32) + + + def build(self): + # Create three convolutional layers with relu activations + self.conv1=conv2d(weight_shape=(3,3,3,32),activation='relu') + self.conv2=conv2d(weight_shape=(3,3,32,64),activation='relu') + self.conv3=conv2d(weight_shape=(3,3,64,64),activation='relu') + # Create two dense layers with relu and linear activations + self.dense1=dense([64*4*4,64],activation='relu') + self.dense2=dense([64,10]) + # Store the parameters of the layers in a list + self.param = [self.conv1.weight, + self.conv2.weight, + self.conv3.weight, + self.dense1.weight, + self.dense1.bias, + self.dense2.weight, + self.dense2.bias] + return + + + def data_func(self,data_batch,labels_batch): + # Apply mixup augmentation on the input data and labels + return lmix(data_batch,labels_batch,self.alpha) + + + def fp(self,data): + # Perform forward propagation on the input data + x=tf.nn.conv2d(data,self.conv1.weight,strides=(1,1),padding='SAME') # First convolutional layer + x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # First max pooling layer + x=tf.nn.conv2d(x,self.conv2.weight,strides=(1,1),padding='SAME') # Second convolutional layer + x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # Second max pooling layer + x=tf.nn.conv2d(x,self.conv3.weight,strides=(1,1),padding='SAME') # Third convolutional layer + x=flatten(x) # Flatten the output to a vector + x=tf.nn.relu(tf.matmul(x,self.dense1.weight)+self.dense1.bias) # First dense layer with relu activation + x=tf.nn.dropout(x,rate=0.5) # Apply dropout to prevent overfitting + output=tf.matmul(x,self.dense2.weight)+self.dense2.bias # Output layer with linear activation + output=tf.nn.dropout(output,rate=0.5) # Apply dropout to prevent overfitting + return output + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) + + + def opt(self,gradient): + # Perform optimization on the parameters using the gradient and the batch count + param=self.optimizer.opt(gradient,self.param,self.bc[0]) + return param \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/process/nn.py b/Note 7.0 documentation/DL/neural network/tensorflow/process/nn.py new file mode 100644 index 00000000..70f2a637 --- /dev/null +++ b/Note 7.0 documentation/DL/neural network/tensorflow/process/nn.py @@ -0,0 +1,39 @@ +import tensorflow as tf +import Note.nn.layer.dense as d +from Note.nn.process.optimizer import Momentum +from Note.nn.layer.flatten import flatten + +# Define a neural network class +class nn: + def __init__(self): + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.optimizer=Momentum(0.07,0.7) + + + def build(self): + # Create two dense layers with relu and linear activations + self.layer1=d.dense([784,128],activation='relu') + self.layer2=d.dense([128,10]) + # Store the parameters of the layers in a list + self.param=[self.layer1.param,self.layer2.param] + return + + + def fp(self,data): + # Perform forward propagation on the input data + data=flatten(data) + output1=self.layer1.output(data) + output2=self.layer2.output(output1) + return output2 + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) + + + def opt(self,gradient): + # Perform optimization on the parameters using the gradient + param=self.optimizer.opt(gradient,self.param) + return param \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/process/nn_.py b/Note 7.0 documentation/DL/neural network/tensorflow/process/nn_.py new file mode 100644 index 00000000..3f50c1e3 --- /dev/null +++ b/Note 7.0 documentation/DL/neural network/tensorflow/process/nn_.py @@ -0,0 +1,39 @@ +import tensorflow as tf +from Note.nn.process.optimizer import Momentum +from Note.nn.layer.flatten import flatten + +# Define a neural network class +class nn: + def __init__(self): + # Initialize the weights and biases of three layers with random values + self.weight1=tf.Variable(tf.random.normal([784,64],dtype='float64')) + self.bias1=tf.Variable(tf.random.normal([64],dtype='float64')) + self.weight2=tf.Variable(tf.random.normal([64,64],dtype='float64')) + self.bias2=tf.Variable(tf.random.normal([64],dtype='float64')) + self.weight3=tf.Variable(tf.random.normal([64,10],dtype='float64')) + self.bias3=tf.Variable(tf.random.normal([10],dtype='float64')) + # Store the parameters of the layers in a list + self.param=[self.weight1,self.weight2,self.weight3,self.bias1,self.bias2,self.bias3] + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.optimizer=Momentum(0.07,0.7) + + + def fp(self,data): + # Perform forward propagation on the input data + data=flatten(data) + layer1=tf.nn.relu(tf.matmul(data,self.weight1)+self.bias1) # First layer with relu activation + layer2=tf.nn.relu(tf.matmul(layer1,self.weight2)+self.bias2) # Second layer with relu activation + output=tf.matmul(layer2,self.weight3)+self.bias3 # Output layer with linear activation + return output + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) + + + def opt(self,gradient): + # Perform optimization on the parameters using the gradient + param=self.optimizer.opt(gradient,self.param) + return param \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/process/nn_acc.py b/Note 7.0 documentation/DL/neural network/tensorflow/process/nn_acc.py new file mode 100644 index 00000000..11f29cda --- /dev/null +++ b/Note 7.0 documentation/DL/neural network/tensorflow/process/nn_acc.py @@ -0,0 +1,44 @@ +import tensorflow as tf +import Note.nn.layer.dense as d +from Note.nn.process.optimizer import Momentum +from Note.nn.layer.flatten import flatten +from Note.nn.accuracy import sparse_categorical_accuracy + +# Define a neural network class +class nn: + def __init__(self): + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.optimizer=Momentum(0.07,0.7) + + + def build(self): + # Create two dense layers with relu and linear activations + self.layer1=d.dense([784,128],activation='relu') + self.layer2=d.dense([128,10]) + # Store the parameters of the layers in a list + self.param=[self.layer1.param,self.layer2.param] + return + + + def fp(self,data): + # Perform forward propagation on the input data + data=flatten(data) + output1=self.layer1.output(data) + output2=self.layer2.output(output1) + return output2 + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) + + + def accuracy(self,output,labels): #accuracy function,kernel uses it to calculate accuracy. + return sparse_categorical_accuracy(labels,output) + + + def opt(self,gradient): + # Perform optimization on the parameters using the gradient + param=self.optimizer.opt(gradient,self.param) + return param \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/process/nn_attenuate.py b/Note 7.0 documentation/DL/neural network/tensorflow/process/nn_attenuate.py new file mode 100644 index 00000000..890b3335 --- /dev/null +++ b/Note 7.0 documentation/DL/neural network/tensorflow/process/nn_attenuate.py @@ -0,0 +1,49 @@ +import tensorflow as tf +import Note.nn.layer.dense as d +from Note.nn.process.optimizer import Momentum +from Note.nn.layer.flatten import flatten + +# Define a neural network class with gradient attenuation +class nn: + def __init__(self): + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.optimizer=Momentum(0.07,0.7) + # Initialize a variable to keep track of the number of optimization steps + self.opt_counter=tf.Variable(tf.zeros(7,dtype=tf.float32)) + + + def build(self): + # Create two dense layers with relu and linear activations + self.layer1=d.dense([784,128],activation='relu') + self.layer2=d.dense([128,10]) + # Store the parameters of the layers in a list + self.param=[self.layer1.param,self.layer2.param] + return + + + def fp(self,data): + # Perform forward propagation on the input data + data=flatten(data) + output1=self.layer1.output(data) + output2=self.layer2.output(output1) + return output2 + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) + + + def attenuate(self,gradient,oc,p): #gradient attenuation function,kernel uses it to calculate attenuation coefficient. + # Apply an exponential decay to the gradient based on the optimization counter + ac=0.9**oc[p] #ac:attenuation coefficient + for i in range(len(gradient)): #oc:optimization counter + gradient[i]=ac*gradient[i] #p:process number + return gradient + + + def opt(self,gradient): + # Perform optimization on the parameters using the gradient + param=self.optimizer.opt(gradient,self.param) + return param \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/process/nn_clipping.py b/Note 7.0 documentation/DL/neural network/tensorflow/process/nn_clipping.py new file mode 100644 index 00000000..32779f84 --- /dev/null +++ b/Note 7.0 documentation/DL/neural network/tensorflow/process/nn_clipping.py @@ -0,0 +1,47 @@ +import tensorflow as tf +import Note.nn.layer.dense as d +from Note.nn.process.optimizer import Momentum +from Note.nn.layer.flatten import flatten +from Note.nn.gradient_clipping import gradient_clipping + +# Define a neural network class +class nn: + def __init__(self): + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.optimizer=Momentum(0.07,0.7) + + + def build(self): + # Create two dense layers with relu and linear activations + self.layer1=d.dense([784,128],activation='relu') + self.layer2=d.dense([128,10]) + # Store the parameters of the layers in a list + self.param=[self.layer1.param,self.layer2.param] + return + + + def fp(self,data): + # Perform forward propagation on the input data + data=flatten(data) + output1=self.layer1.output(data) + output2=self.layer2.output(output1) + return output2 + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) + + + def gradient(self,tape,loss): + # Compute the gradients of the loss with respect to the parameters + grads=tape.gradient(loss,self.param) + # Clip the gradients by value to avoid exploding gradients + return gradient_clipping(grads,'value',1.0) + + + def opt(self,gradient): + # Perform optimization on the parameters using the gradient + param=self.optimizer.opt(gradient,self.param) + return param \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/process/self_LSTM.py b/Note 7.0 documentation/DL/neural network/tensorflow/process/self_LSTM.py new file mode 100644 index 00000000..a25f4043 --- /dev/null +++ b/Note 7.0 documentation/DL/neural network/tensorflow/process/self_LSTM.py @@ -0,0 +1,54 @@ +import tensorflow as tf +from Note.nn.process.optimizer import Adam +from Note.nn.layer.LSTM import LSTM +from Note.nn.layer.self_attention import self_attention +from Note.nn.layer.dense import dense +from Note.nn.layer.emdedding import embedding + +# Define a LSTM self attention neural network class +class self_LSTM: + def __init__(self,vocab_size,embed_size,num_heads,num_layers): + self.vocab_size=vocab_size + self.embed_size=embed_size + self.num_heads=num_heads + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.optimizer=Adam() + # Initialize a variable to keep track of the batch count + self.bc=tf.Variable(0,dtype=tf.float32) + + + def build(self): + # Create an embedding layer to map input tokens to vectors + self.embedding_layer = embedding(self.vocab_size,self.embed_size) + # Create a LSTM layer to process the input sequence + self.lstm_layer = LSTM(weight_shape=[self.embed_size,self.embed_size],return_sequence=True) + # Create a self attention layer to attend to the LSTM output + self.self_attention_layer = self_attention(weight_shape=[self.embed_size,self.embed_size],num_heads=self.num_heads) + # Create a linear layer to map the attention output to logits + self.logits_layer = dense([self.embed_size,self.vocab_size]) + # Store the parameters of the layers in a list + self.param = [self.embedding_layer.embeddings] + self.param.extend(self.lstm_layer.param) + self.param.extend(self.self_attention_layer.param) + self.param.extend(self.logits_layer.param) + + + def fp(self,data): + # Perform forward propagation on the input data + x = self.embedding_layer.output(data) # shape: (batch_size, seq_len, embed_size) + x = self.lstm_layer.output(x) # shape: (batch_size, seq_len, embed_size) + x, attention_weights = self.self_attention_layer.output(x,self.num_heads) # shape: (batch_size, seq_len, embed_size) + output = self.logits_layer.output(x) # shape: (batch_size, seq_len, vocab_size) + return output + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) + + + def opt(self,gradient): + # Perform optimization on the parameters using the gradient and the batch count + param=self.optimizer.opt(gradient,self.param,self.bc[0]) + return param \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/process/transformer.py b/Note 7.0 documentation/DL/neural network/tensorflow/process/transformer.py new file mode 100644 index 00000000..a77978a6 --- /dev/null +++ b/Note 7.0 documentation/DL/neural network/tensorflow/process/transformer.py @@ -0,0 +1,55 @@ +import tensorflow as tf +from Note.nn.process.optimizer import Adam +from Note.nn.layer.transformer import transformer +from Note.nn.layer.dense import dense +from Note.nn.layer.emdedding import embedding +from Note.nn.positional_encoding import positional_encoding + +# Define a positional encoding transformer neural network class +class Transformer: + def __init__(self,vocab_size,embed_size,num_heads,num_layers,max_len): + self.vocab_size=vocab_size + self.embed_size=embed_size + self.num_heads=num_heads + self.num_layers=num_layers + self.max_len=max_len + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.optimizer=Adam() + # Initialize a variable to keep track of the batch count + self.bc=tf.Variable(0,dtype=tf.float32) + + + def build(self): + # Create an embedding layer to map input tokens to vectors + self.embedding_layer = embedding(self.vocab_size,self.embed_size) + # Create a list of transformer layers + self.transformer_layers = [transformer(weight_shape=[self.embed_size, self.embed_size], num_heads=self.num_heads) for _ in range(self.num_layers)] + # Create a linear layer to map the transformer output to logits + self.logits_layer = dense([self.embed_size,self.vocab_size]) + # Store the parameters of the layers in a list + self.param = [self.embedding_layer.embeddings] + for transformer_layer in self.transformer_layers: + self.param.extend(transformer_layer.param) + self.param.extend(self.logits_layer.param) + + + def fp(self,data): + # Perform forward propagation on the input data + x = self.embedding_layer.output(data) # shape: (batch_size, seq_len, embed_size) + x += positional_encoding(self.max_len,self.embed_size) # shape: (batch_size, seq_len, embed_size) + for transformer_layer in self.transformer_layers: + x = transformer_layer.output(x) # shape: (batch_size, seq_len, embed_size) + output = self.logits_layer.output(x) # shape: (batch_size, seq_len, vocab_size) + return output + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) + + + def opt(self,gradient): + # Perform optimization on the parameters using the gradient and the batch count + param=self.optimizer.opt(gradient,self.param,self.bc[0]) + return param \ No newline at end of file diff --git a/Note 7.0 documentation/DL/reduced kernel/kernel_reduced.py b/Note 7.0 documentation/DL/reduced kernel/kernel_reduced.py new file mode 100644 index 00000000..392d5897 --- /dev/null +++ b/Note 7.0 documentation/DL/reduced kernel/kernel_reduced.py @@ -0,0 +1,741 @@ +from tensorflow import function +import numpy as np +import matplotlib.pyplot as plt +import time + + +class kernel: + def __init__(self,nn=None): + self.nn=nn # the neural network object + try: + self.nn.km=1 # a flag to indicate the kernel mode + except Exception: + pass + self.platform=None # the platform to use, either tensorflow or pytorch + self.batches=None # the number of batches for training data + self.suspend=False # a flag to indicate whether to suspend the training + self.stop=False # a flag to indicate whether to stop the training + self.stop_flag=False # a flag to indicate whether the training has been stopped + self.save_epoch=None # the epoch number to save the model + self.batch=None # the batch size for training data + self.epoch=0 # the current epoch number + self.end_loss=None # the target loss value to end the training + self.end_acc=None # the target accuracy value to end the training + self.end_test_loss=None # the target test loss value to end the training + self.end_test_acc=None # the target test accuracy value to end the training + self.acc_flag='%' # a flag to indicate whether to use percentage or decimal for accuracy display + self.train_counter=0 # a counter for how many times the train method has been called + self.filename='save.dat' # the file name to save the model + self.train_loss=None # the current train loss value + self.train_acc=None # the current train accuracy value + self.train_loss_list=[] # a list of train loss values for each epoch + self.train_acc_list=[] # a list of train accuracy values for each epoch + self.test_loss=None # the current test loss value + self.test_acc=None # the current test accuracy value + self.test_loss_list=[] # a list of test loss values for each epoch + self.test_acc_list=[] # a list of test accuracy values for each epoch + self.test_flag=False # a flag to indicate whether to use test data or not + self.total_epoch=0 # the total number of epochs for all trainings + self.time=0 # the time elapsed for one training session + self.total_time=0 # the total time elapsed for all trainings + + + def data(self,train_data=None,train_labels=None,test_data=None,test_labels=None,train_dataset=None,test_dataset=None): + if type(self.nn.param[0])!=list: + self.train_data=train_data.astype(self.nn.param[0].dtype.name) # convert the train data type to match the neural network parameters type + self.train_labels=train_labels.astype(self.nn.param[0].dtype.name) # convert the train labels type to match the neural network parameters type + else: + self.train_data=train_data.astype(self.nn.param[0][0].dtype.name) # convert the train data type to match the neural network parameters type (for multiple inputs) + self.train_labels=train_labels.astype(self.nn.param[0][0].dtype.name) # convert the train labels type to match the neural network parameters type (for multiple inputs) + self.train_dataset=train_dataset # a tensorflow or pytorch dataset object for train data (optional) + if test_data is not None: + if type(self.nn.param[0])!=list: + self.test_data=test_data.astype(self.nn.param[0].dtype.name) # convert the test data type to match the neural network parameters type + self.test_labels=test_labels.astype(self.nn.param[0].dtype.name) # convert the test labels type to match the neural network parameters type + else: + self.test_data=test_data.astype(self.nn.param[0][0].dtype.name) # convert the test data type to match the neural network parameters type (for multiple inputs) + self.test_labels=test_labels.astype(self.nn.param[0][0].dtype.name) # convert the test labels type to match the neural network parameters type (for multiple inputs) + self.test_flag=True # set the test flag to True if test data is provided + self.test_dataset=test_dataset # a tensorflow or pytorch dataset object for test data (optional) + if self.train_dataset==None: + self.shape0=train_data.shape[0] # get the number of samples in train data + return + + + def init(self): # a method to initialize the attributes for a new training session + self.suspend=False + self.stop=False + self.stop_flag=False + self.save_epoch=None + self.end_loss=None + self.end_acc=None + self.end_test_loss=None + self.end_test_acc=None + self.train_loss=None + self.train_acc=None + self.test_loss=None + self.test_acc=None + self.train_loss_list.clear() + self.train_acc_list.clear() + self.test_loss_list.clear() + self.test_acc_list.clear() + self.test_flag=False + self.train_counter=0 + self.epoch=0 + self.total_epoch=0 + self.time=0 + self.total_time=0 + return + + + def end(self): # a method to check whether the training has reached the target loss or accuracy values + if self.end_loss!=None and len(self.train_loss_list)!=0 and self.train_loss_list[-1]self.end_acc: # if the target train accuracy is given and the current train accuracy is higher than it + return True + elif self.end_loss!=None and len(self.train_loss_list)!=0 and self.end_acc!=None and self.train_loss_list[-1]self.end_acc: # if both the target train loss and accuracy are given and the current train loss is lower than it and the current train accuracy is higher than it + return True + elif self.end_test_loss!=None and len(self.test_loss_list)!=0 and self.test_loss_list[-1]self.end_test_acc: # if the target test accuracy is given and the current test accuracy is higher than it + return True + elif self.end_test_loss!=None and len(self.test_loss_list)!=0 and self.end_test_acc!=None and self.test_loss_list[-1]self.end_test_acc: # if both the target test loss and accuracy are given and the current test loss is lower than it and the current test accuracy is higher than it + return True + + + def loss_acc(self,output=None,labels_batch=None,loss=None,test_batch=None,total_loss=None,total_acc=None): # a method to calculate the loss and accuracy values for each batch or epoch + if self.batch!=None: # if batch mode is used + total_loss+=loss # accumulate the batch loss to total loss + try: + batch_acc=self.nn.accuracy(output,labels_batch) # calculate the batch accuracy using the neural network's accuracy method + total_acc+=batch_acc # accumulate the batch accuracy to total accuracy + except Exception as e: + try: + if self.nn.accuracy!=None: # if the neural network has an accuracy method defined + raise e # raise the exception + except Exception: + pass + return total_loss,total_acc # return the total loss and accuracy values for all batches so far + else: # if batch mode is not used (use all data at once) + loss=loss.numpy() # convert the loss value to numpy array + self.train_loss=loss # assign the loss value to train loss attribute + self.train_loss_list.append(loss) # append the loss value to train loss list + try: + acc=self.nn.accuracy(output,self.train_labels) # calculate the accuracy value using the neural network's accuracy method + acc=acc.numpy() # convert the accuracy value to numpy array + self.train_acc=acc # assign the accuracy value to train accuracy attribute + self.train_acc_list.append(acc) # append the accuracy value to train accuracy list + except Exception as e: + try: + if self.nn.accuracy!=None: # if the neural network has an accuracy method defined + raise e # raise the exception + except Exception: + pass + if self.test_flag==True: # if test data is used + self.test_loss,self.test_acc=self.test(self.test_data,self.test_labels,test_batch) # calculate the test loss and accuracy values using the test method + self.test_loss_list.append(self.test_loss) # append the test loss value to test loss list + try: + self.test_acc_list.append(self.test_acc) # append the test accuracy value to test accuracy list + except Exception as e: + try: + if self.nn.accuracy!=None: # if the neural network has an accuracy method defined + raise e # raise the exception + except Exception: + pass + return # return nothing for this case + + + def data_func(self,data_batch=None,labels_batch=None,batch=None,index1=None,index2=None,j=None,flag=None): # a method to get a batch of data and labels from the train data and labels + if flag==None: # if flag is None, it means the batch size is smaller than the number of samples + if batch!=1: # if batch size is not 1 + data_batch=self.train_data[index1:index2] # get a slice of train data according to the index range + else: # if batch size is 1 + data_batch=self.train_data[j] # get one sample of train data according to the index + if batch!=1: # if batch size is not 1 + labels_batch=self.train_labels[index1:index2] # get a slice of train labels according to the index range + else: # if batch size is 1 + labels_batch=self.train_labels[j] # get one sample of train labels according to the index + else: # if flag is not None, it means the batch size is larger than the number of samples + try: + try: + data_batch=self.platform.concat([self.train_data[index1:],self.train_data[:index2]],0) # concatenate two slices of train data from the end and the beginning to form a batch + labels_batch=self.platform.concat([self.train_labels[index1:],self.train_labels[:index2]],0) # concatenate two slices of train labels from the end and the beginning to form a batch + except Exception: # if the platform's concat method fails + data_batch=np.concatenate([self.train_data[index1:],self.train_data[:index2]],0) # use numpy's concatenate method instead + labels_batch=np.concatenate([self.train_labels[index1:],self.train_labels[:index2]],0) # use numpy's concatenate method instead + except Exception as e: + raise e # raise any other exception + return data_batch,labels_batch # return the batch of data and labels + + + @function(jit_compile=True) # use tensorflow's function decorator to speed up the execution + def tf_opt(self,data,labels): # a method to perform one optimization step using tensorflow platform + try: + try: + if self.nn.GradientTape!=None: # if the neural network has a GradientTape method defined + tape,output,loss=self.nn.GradientTape(data,labels) # use the neural network's GradientTape method to get the tape, output and loss values + except Exception: # if the neural network does not have a GradientTape method defined or it fails + with self.platform.GradientTape(persistent=True) as tape: # use tensorflow's GradientTape context manager instead + try: + output=self.nn.fp(data) # get the output value using the neural network's forward propagation method + loss=self.nn.loss(output,labels) # get the loss value using the neural network's loss function + except Exception: # if the neural network's forward propagation method or loss function fails or they are combined in one method + output,loss=self.nn.fp(data,labels) # use the neural network's forward propagation method with both data and labels as inputs to get the output and loss values + except Exception as e: + raise e # raise any other exception + try: + try: + gradient=self.nn.gradient(tape,loss) # use the neural network's gradient method to get the gradient value from the tape and loss values + except Exception: # if the neural network does not have a gradient method defined or it fails + gradient=tape.gradient(loss,self.nn.param) # use tensorflow's tape.gradient method instead with loss value and neural network parameters as inputs + except Exception as e: + raise e # raise any other exception + try: + try: + self.nn.opt.apply_gradients(zip(gradient,self.nn.param)) # use the neural network's optimizer's apply_gradients method to update the neural network parameters with gradient value + except Exception: # if the neural network does not have an optimizer or its apply_gradients method fails + self.nn.opt(gradient) # use the neural network's optimizer directly with gradient value as input + except Exception as e: + raise e # raise any other exception + return output,loss # return the output and loss values for this optimization step + + + def pytorch_opt(self,data,labels): # a method to perform one optimization step using pytorch platform + output=self.nn.fp(data) # get the output value using the neural network's forward propagation method + loss=self.nn.loss(output,labels) # get the loss value using the neural network's loss function + try: + try: + self.nn.opt.zero_grad() # use the neural network's optimizer's zero_grad method to clear the previous gradients + loss.backward() # use pytorch's loss.backward method to calculate the gradients + self.nn.opt.step() # use the neural network's optimizer's step method to update the neural network parameters with gradient value + except Exception: # if the neural network does not have an optimizer or its zero_grad or step method fails + self.nn.opt(loss) # use the neural network's optimizer directly with loss value as input + except Exception as e: + raise e # raise any other exception + return output,loss # return the output and loss values for this optimization step + + + def opt(self,data,labels): # a method to perform one optimization step using either tensorflow or pytorch platform + try: + try: + if self.platform.DType!=None: # if tensorflow platform is used + output,loss=self.tf_opt(data,labels) # use the tf_opt method for optimization + except Exception: # if tensorflow platform is not used or it fails + output,loss=self.pytorch_opt(data,labels) # use the pytorch_opt method for optimization + except Exception as e: + raise e # raise any other exception + return output,loss # return the output and loss values for this optimization step + + + def _train(self,batch=None,test_batch=None): # a method to perform one epoch of training + if batch!=None: # if batch mode is used + total_loss=0 # initialize the total loss value for all batches + total_acc=0 # initialize the total accuracy value for all batches + if self.train_dataset!=None: # if a tensorflow or pytorch dataset object is used for train data + for data_batch,labels_batch in self.train_dataset: # iterate over each batch of data and labels from the train dataset + if self.stop==True: # if the stop flag is set to True + if self.stop_func(): # check whether the training has reached the target loss or accuracy values using the stop_func method + return # return nothing and end the training + try: + data_batch,labels_batch=self.nn.data_func(data_batch,labels_batch) # use the neural network's data_func method to preprocess the data and labels batch (optional) + except Exception as e: + try: + if self.nn.data_func!=None: # if the neural network has a data_func method defined + raise e # raise the exception + except Exception: + pass + output,batch_loss=self.opt(data_batch,labels_batch) # perform one optimization step using the opt method and get the output and batch loss values + total_loss,total_acc=self.loss_acc(output=output,labels_batch=labels_batch,loss=batch_loss,total_loss=total_loss,total_acc=total_acc) # calculate and update the total loss and accuracy values for all batches using the loss_acc method + else: # if a numpy array is used for train data + total_loss=0 # initialize the total loss value for all batches + total_acc=0 # initialize the total accuracy value for all batches + batches=int((self.shape0-self.shape0%batch)/batch) # calculate how many batches are needed for train data according to batch size + for j in range(batches): # iterate over each batch index + if self.stop==True: # if the stop flag is set to True + if self.stop_func(): # check whether the training has reached the target loss or accuracy values using the stop_func method + return # return nothing and end the training + index1=j*batch # calculate the start index of train data for this batch + index2=(j+1)*batch # calculate the end index of train data for this batch + data_batch,labels_batch=self.data_func(batch,index1,index2,j) # get a batch of data and labels from train data and labels using the data_func method + try: + data_batch,labels_batch=self.nn.data_func(data_batch,labels_batch) # use the neural network's data_func method to preprocess the data and labels batch (optional) + except Exception as e: + try: + if self.nn.data_func!=None: # if the neural network has a data_func method defined + raise e # raise the exception + except Exception: + pass + output,batch_loss=self.opt(data_batch,labels_batch) # perform one optimization step + total_loss,total_acc=self.loss_acc(output=output,labels_batch=labels_batch,loss=batch_loss,total_loss=total_loss,total_acc=total_acc) # calculate and update the total loss and accuracy values for all batches using the loss_acc method + try: + try: + self.nn.bc.assign_add(1) # use the neural network's batch counter's assign_add method to increase the batch counter by 1 (for tensorflow platform) + except Exception: # if the neural network does not have a batch counter or its assign_add method fails + self.nn.bc+=1 # use the normal addition operation to increase the batch counter by 1 (for pytorch platform) + except Exception: + pass + if self.shape0%batch!=0: # if there are some samples left in train data that are not enough to form a full batch + if self.stop==True: # if the stop flag is set to True + if self.stop_func(): # check whether the training has reached the target loss or accuracy values using the stop_func method + return # return nothing and end the training + batches+=1 # increase the number of batches by 1 to include the remaining samples + index1=batches*batch # calculate the start index of train data for the last batch + index2=batch-(self.shape0-batches*batch) # calculate how many samples are needed from the beginning of train data to form a full batch + data_batch,labels_batch=self.data_func(batch,index1,index2,flag=True) # get a batch of data and labels from train data and labels using the data_func method with flag set to True + try: + data_batch,labels_batch=self.nn.data_func(data_batch,labels_batch) # use the neural network's data_func method to preprocess the data and labels batch (optional) + except Exception as e: + try: + if self.nn.data_func!=None: # if the neural network has a data_func method defined + raise e # raise the exception + except Exception: + pass + output,batch_loss=self.opt(data_batch,labels_batch) # perform one optimization step using the opt method and get the output and batch loss values + total_loss,total_acc=self.loss_acc(output=output,labels_batch=labels_batch,loss=batch_loss,total_loss=total_loss,total_acc=total_acc) # calculate and update the total loss and accuracy values for all batches using the loss_acc method + try: + try: + self.nn.bc.assign_add(1) # use the neural network's batch counter's assign_add method to increase the batch counter by 1 (for tensorflow platform) + except Exception: # if the neural network does not have a batch counter or its assign_add method fails + self.nn.bc+=1 # use the normal addition operation to increase the batch counter by 1 (for pytorch platform) + except Exception: + pass + try: + if self.platform.DType!=None: # if tensorflow platform is used + loss=total_loss.numpy()/batches # convert the total loss value to numpy array and divide it by number of batches to get the average loss value for this epoch + except Exception: # if tensorflow platform is not used or it fails + loss=total_loss.detach().numpy()/batches # detach the total loss value from computation graph and convert it to numpy array and divide it by number of batches to get the average loss value for this epoch + try: + train_acc=total_acc.numpy()/batches # convert the total accuracy value to numpy array and divide it by number of batches to get the average accuracy value for this epoch + except Exception as e: + try: + if self.nn.accuracy!=None: # if the neural network has an accuracy method defined + raise e # raise the exception + except Exception: + pass + self.train_loss=loss # assign the average loss value to train loss attribute + self.train_loss_list.append(loss) # append the average loss value to train loss list + try: + self.train_acc=train_acc # assign the average accuracy value to train accuracy attribute + self.train_acc_list.append(train_acc) # append the average accuracy value to train accuracy list + except Exception as e: + try: + if self.nn.accuracy!=None: # if the neural network has an accuracy method defined + raise e # raise the exception + except Exception: + pass + if self.test_flag==True: # if test data is used + self.test_loss,self.test_acc=self.test(self.test_data,self.test_labels,test_batch) # calculate the test loss and accuracy values using the test method + self.test_loss_list.append(self.test_loss) # append the test loss value to test loss list + try: + self.test_acc_list.append(self.test_acc) # append the test accuracy value to test accuracy list + except Exception as e: + try: + if self.nn.accuracy!=None: # if the neural network has an accuracy method defined + raise e # raise the exception + except Exception: + pass + else: # if batch mode is not used (use all data at once) + output,train_loss=self.opt(self.train_data,self.train_labels) # perform one optimization step using the opt method and get the output and train loss values + self.loss_acc(output=output,labels_batch=labels_batch,loss=train_loss,test_batch=test_batch,total_loss=total_loss,total_acc=total_acc) # calculate and update the train and test loss and accuracy values using the loss_acc method + return # return nothing for this case + + + def train(self,batch=None,epoch=None,test_batch=None,save=None,one=True,p=None,s=None): # a method to perform multiple epochs of training + self.batch=batch # assign the batch size for training data to batch attribute + self.epoch=0 # initialize the current epoch number to 0 + self.train_counter+=1 # increase the train counter by 1 + if p==None: # if p is None, it means the default value of p is used + self.p=9 # assign 9 to p attribute, which means print the train and test loss and accuracy values every 10 epochs + else: + self.p=p-1 # assign p-1 to p attribute, which means print the train and test loss and accuracy values every p epochs + if s==None: # if s is None, it means the default value of s is used + self.s=1 # assign 1 to s attribute, which means save the model every epoch + self.file_list=None # assign None to file_list attribute, which means do not keep track of saved files + else: + self.s=s-1 # assign s-1 to s attribute, which means save the model every s epochs + self.file_list=[] # assign an empty list to file_list attribute, which means keep track of saved files + if epoch!=None: # if epoch is not None, it means a fixed number of epochs is given + for i in range(epoch): # iterate over each epoch index + t1=time.time() # record the start time of this epoch + self._train(batch,test_batch) # perform one epoch of training using the _train method + if self.stop_flag==True: # if the stop flag is set to True, it means the training has reached the target loss or accuracy values and has been stopped + return # return nothing and end the training + try: + try: + self.nn.ec.assign_add(1) # use the neural network's epoch counter's assign_add method to increase the epoch counter by 1 (for tensorflow platform) + except Exception: # if the neural network does not have an epoch counter or its assign_add method fails + self.nn.ec+=1 # use the normal addition operation to increase the epoch counter by 1 (for pytorch platform) + except Exception: + pass + self.total_epoch+=1 # increase the total epoch number by 1 for all trainings + if epoch%10!=0: + p=epoch-epoch%self.p + p=int(p/self.p) + s=epoch-epoch%self.s + s=int(s/self.s) + else: + p=epoch/(self.p+1) + p=int(p) + s=epoch/(self.s+1) + s=int(s) + if p==0: + p=1 + if s==0: + s=1 + if i%p==0: # if this epoch index is a multiple of p (or p+1) + if self.test_flag==False: # if test data is not used + try: + if self.nn.accuracy!=None: # if the neural network has an accuracy method defined + print('epoch:{0} loss:{1:.6f}'.format(i+1,self.train_loss)) # print the current epoch number and train loss value + if self.acc_flag=='%': # if percentage mode is used for accuracy display + print('epoch:{0} accuracy:{1:.1f}'.format(i+1,self.train_acc*100)) # print the current epoch number and train accuracy value in percentage + else: # if decimal mode is used for accuracy display + print('epoch:{0} accuracy:{1:.6f}'.format(i+1,self.train_acc)) # print the current epoch number and train accuracy value in print('epoch:{0} accuracy:{1:.6f}'.format(i+1,self.train_acc)) # print the current epoch number and train accuracy value in decimal + print() # print an empty line for readability + except Exception: # if the neural network does not have an accuracy method defined or it fails + print('epoch:{0} loss:{1:.6f}'.format(i+1,self.train_loss)) # print the current epoch number and train loss value + print() # print an empty line for readability + else: # if test data is used + try: + if self.nn.accuracy!=None: # if the neural network has an accuracy method defined + print('epoch:{0} loss:{1:.6f},test loss:{2:.6f}'.format(i+1,self.train_loss,self.test_loss)) # print the current epoch number, train loss value and test loss value + if self.acc_flag=='%': # if percentage mode is used for accuracy display + print('epoch:{0} accuracy:{1:.1f},test accuracy:{2:.1f}'.format(i+1,self.train_acc*100,self.test_acc*100)) # print the current epoch number, train accuracy value and test accuracy value in percentage + else: # if decimal mode is used for accuracy display + print('epoch:{0} accuracy:{1:.6f},test accuracy:{2:.6f}'.format(i+1,self.train_acc,self.test_acc)) # print the current epoch number, train accuracy value and test accuracy value in decimal + print() # print an empty line for readability + except Exception: # if the neural network does not have an accuracy method defined or it fails + print('epoch:{0} loss:{1:.6f},test loss:{2:.6f}'.format(i+1,self.train_loss,self.test_loss)) # print the current epoch number, train loss value and test loss value + print() # print an empty line for readability + if save!=None and i%s==0: # if save is not None, it means a file name is given to save the model, and if this epoch index is a multiple of s (or s+1) + self.save(self.total_epoch,one) # save the model using the save method with total epoch number and one flag as inputs + t2=time.time() # record the end time of this epoch + self.time+=(t2-t1) # calculate and update the time elapsed for this training session + else: # if epoch is None, it means an infinite number of epochs is given + i=0 # initialize the epoch index to 0 + while True: # loop indefinitely until stopped by other conditions + t1=time.time() # record the start time of this epoch + self._train(test_batch=test_batch) # perform one epoch of training using the _train method + if self.stop_flag==True: # if the stop flag is set to True, it means the training has reached the target loss or accuracy values and has been stopped + return # return nothing and end the training + i+=1 # increase the epoch index by 1 + try: + try: + self.nn.ec.assign_add(1) # use the neural network's epoch counter's assign_add method to increase the epoch counter by 1 (for tensorflow platform) + except Exception: # if the neural network does not have an epoch counter or its assign_add method fails + self.nn.ec+=1 # use the normal addition operation to increase the epoch counter by 1 (for pytorch platform) + except Exception: + pass + self.total_epoch+=1 # increase the total epoch number by 1 for all trainings + if epoch%10!=0: + p=epoch-epoch%self.p + p=int(p/self.p) + s=epoch-epoch%self.s + s=int(s/self.s) + else: + p=epoch/(self.p+1) + p=int(p) + s=epoch/(self.s+1) + s=int(s) + if p==0: + p=1 + if s==0: + s=1 + if i%p==0: # if this epoch index is a multiple of p (or p+1) + if self.test_flag==False: # if test data is not used + try: + if self.nn.accuracy!=None: # if the neural network has an accuracy method defined + print('epoch:{0} loss:{1:.6f}'.format(i+1,self.train_loss)) # print the current epoch number and train loss value + if self.acc_flag=='%': # if percentage mode is used for accuracy display + print('epoch:{0} accuracy:{1:.1f}'.format(i+1,self.train_acc*100)) # print the current epoch number and train accuracy value in percentage + else: # if decimal mode is used for accuracy display + print('epoch:{0} accuracy:{1:.6f}'.format(i+1,self.train_acc)) # print the current epoch number and train accuracy value in decimal + print() # print an empty line for readability + except Exception: # if the neural network does not have an accuracy method defined or it fails + print('epoch:{0} loss:{1:.6f}'.format(i+1,self.train_loss)) # print the current epoch number and train loss value + print() # print an empty line for readability + else: # if test data is used + try: + if self.nn.accuracy!=None: # if the neural network has an accuracy method defined + print('epoch:{0} loss:{1:.6f},test loss:{2:.6f}'.format(i+1,self.train_loss,self.test_loss)) # print the current epoch number, train loss value and test loss value + if self.acc_flag=='%': # if percentage mode is used for accuracy display + print('epoch:{0} accuracy:{1:.1f},test accuracy:{2:.1f}'.format(i+1,self.train_acc*100,self.test_acc*100)) # print the current epoch number, train accuracy value and test accuracy value in percentage + else: # if decimal mode is used for accuracy display + print('epoch:{0} accuracy:{1:.6f},test accuracy:{2:.6f}'.format(i+1,self.train_acc,self.test_acc)) # print the current epoch number, train accuracy value and test accuracy value in decimal + print() # print an empty line for readability + except Exception: # if the neural network does not have an accuracy method defined or it fails + print('epoch:{0} loss:{1:.6f},test loss:{2:.6f}'.format(i+1,self.train_loss,self.test_loss)) # print the current epoch number, train loss value and test loss value + print() # print an empty line for readability + if save!=None and i%s==0: # if save is not None, it means a file name is given to save the model, and if this epoch index is a multiple of s (or s+1) + self.save(self.total_epoch,one) # save the model using the save method with total epoch number and one flag as inputs + t2=time.time() # record the end time of this epoch + self.time+=(t2-t1) # calculate and update the time elapsed for this training session + if save!=None: # if save is not None, it means a file name is given to save the model + self.save() # save the model using the save method without any inputs (use default values) + self._time=self.time-int(self.time) # get the fractional part of the time elapsed for this training session + if self._time<0.5: # if the fractional part is less than 0.5 + self.time=int(self.time) # round down the time elapsed to integer + else: # if the fractional part is greater than or equal to 0.5 + self.time=int(self.time)+1 # round up the time elapsed to integer + self.total_time+=self.time # calculate and update the total time elapsed for all trainings + if self.test_flag==False: # if test data is not used + print('last loss:{0:.6f}'.format(self.train_loss)) # print the last train loss value + else: # if test data is used + print('last loss:{0:.6f},last test loss:{1:.6f}'.format(self.train_loss,self.test_loss)) # print the last train loss value and test loss value + try: + if self.nn.accuracy!=None: # if the neural network has an accuracy method defined + if self.acc_flag=='%': # if percentage mode is used for accuracy display + if self.test_flag==False: # if test data is not used + print('last accuracy:{0:.1f}'.format(self.train_acc*100)) # print the last train accuracy value in percentage + else: # if test else: # if test data is used + print('last accuracy:{0:.1f},last test accuracy:{1:.1f}'.format(self.train_acc*100,self.test_acc*100)) # print the last train accuracy value and test accuracy value in percentage + else: # if decimal mode is used for accuracy display + if self.test_flag==False: # if test data is not used + print('last accuracy:{0:.6f}'.format(self.train_acc)) # print the last train accuracy value in decimal + else: # if test data is used + print('last accuracy:{0:.6f},last test accuracy:{1:.6f}'.format(self.train_acc,self.test_acc)) # print the last train accuracy value and test accuracy value in decimal + except Exception: # if the neural network does not have an accuracy method defined or it fails + pass + print() # print an empty line for readability + print('time:{0}s'.format(self.time)) # print the time elapsed for this training session + self.training_flag=False # set the training flag to False, which means the training is finished + return # return nothing and end the training + + + def train_ol(self): # a method to perform online learning, which means updating the model with one sample at a time + while True: # loop indefinitely until stopped by other conditions + if self.stop_flag==True: # if the stop flag is set to True, it means the online learning has reached the target loss or accuracy values and has been stopped + return # return nothing and end the online learning + if self.save_flag==True: # if the save flag is set to True, it means a request to save the model has been made + self.save() # save the model using the save method without any inputs (use default values) + self.suspend_func() # check whether a request to suspend the online learning has been made using the suspend_func method + data=self.nn.ol() # get one sample of data and label from the neural network's ol method, which should be defined by the user + if data=='stop': # if the data is 'stop', it means a request to stop the online learning has been made + return # return nothing and end the online learning + elif data=='suspend': # if the data is 'suspend', it means a request to suspend the online learning has been made + self.nn.suspend=True # set the neural network's suspend attribute to True, which means the online learning is suspended + while True: # loop indefinitely until resumed by other conditions + if self.nn.suspend==False: # if the neural network's suspend attribute is set to False, it means a request to resume the online learning has been made + break # break the loop and resume the online learning + continue # continue to the next iteration of online learning + output,loss=self.opt(data[0],data[1]) # perform one optimization step using the opt method and get the output and loss values + loss=loss.numpy() # convert the loss value to numpy array + if len(self.nn.train_loss_list)==self.nn.max_length: # if the train loss list has reached its maximum length, which should be defined by the user + del self.nn.train_loss_list[0] # delete the first element of train loss list + self.nn.train_loss_list.append(loss) # append the loss value to train loss list + try: + train_acc=self.nn.accuracy(output,data[1]) # calculate the accuracy value using the neural network's accuracy method + if len(self.nn.train_acc_list)==self.nn.max_length: # if the train accuracy list has reached its maximum length, which should be defined by the user + del self.nn.train_acc_list[0] # delete the first element of train accuracy list + self.train_acc_list.append(train_acc) # append the accuracy value to train accuracy list + except Exception as e: + try: + if self.nn.accuracy!=None: # if the neural network has an accuracy method defined + raise e # raise the exception + except Exception: + pass + try: + self.nn.c+=1 # increase the neural network's counter by 1, which should be defined by the user to keep track of how many samples have been used for online learning + except Exception: + pass + return # return nothing and end the online learning + + + def test(self,test_data=None,test_labels=None,batch=None): # a method to calculate the test loss and accuracy values using test data and labels + if batch!=None: # if batch mode is used + total_loss=0 # initialize the total loss value for all batches + total_acc=0 # initialize the total accuracy value for all batches + if self.test_dataset!=None: # if a tensorflow or pytorch dataset object is used for test data + for data_batch,labels_batch in self.test_dataset: # iterate over each batch of data and labels from the test dataset + output=self.nn.fp(data_batch) # get the output value using the neural network's forward propagation method + batch_loss=self.nn.loss(output,labels_batch) # get the batch loss value using the neural network's loss function + total_loss+=batch_loss # accumulate the batch loss to total loss + try: + batch_acc=self.nn.accuracy(output,labels_batch) # calculate the batch accuracy using the neural network's accuracy method + total_acc+=batch_acc # accumulate the batch accuracy to total accuracy + except Exception as e: + try: + if self.nn.accuracy!=None: # if the neural network has an accuracy method defined + raise e # raise the exception + except Exception: + pass + else: # if a numpy array is used for test data + total_loss=0 # initialize the total loss value for all batches + total_acc=0 # initialize the total accuracy value for all batches + if type(test_data)==list: # if test data is a list of arrays (for multiple inputs) + batches=int((test_data[0].shape[0]-test_data[0].shape[0]%batch)/batch) # calculate how many batches are needed for test data according to batch size + shape0=test_data[0].shape[0] # get the number of samples in test data + else: # if test data is a single array + batches=int((test_data.shape[0]-test_data.shape[0]%batch)/batch) # calculate how many batches are needed for test data according to batch size + shape0=test_data.shape[0] # get the number of samples in test data + for j in range(batches): # iterate over each batch index + index1=j*batch # calculate the start index of test data for this batch + index2=(j+1)*batch # calculate the end index of test data for this batch + if type(test_data)==list: # if test data is a list of arrays (for multiple inputs) + for i in range(len(test_data)): + data_batch[i]=test_data[i][index1:index2] # get a slice of test data according to the index range for each input array + else: # if test data is a single array + data_batch=test_data[index1:index2] # get a slice of test data according to the index range + if type(test_labels)==list: # if test labels is a list of arrays (for multiple outputs) + for i in range(len(test_labels)): + labels_batch[i]=test_labels[i][index1:index2] # get a slice of test labels according to the index range for each output array + else: # if test labels is a single array + labels_batch=test_labels[index1:index2] # get a slice of test labels according to the index range + output=self.nn.fp(data_batch) # get the output value using the neural network's forward propagation method + batch_loss=self.nn.loss(output,labels_batch) # get the batch loss value using the neural network's loss function + total_loss+=batch_loss # accumulate the batch loss to total loss + try: + batch_acc=self.nn.accuracy(output,labels_batch) # calculate the batch accuracy using the neural network's accuracy method + total_acc+=batch_acc # accumulate the batch accuracy to total accuracy + except Exception as e: + try: + if self.nn.accuracy!=None: # if the neural network has an accuracy method defined + raise e # raise the exception + except Exception: + pass + if shape0%batch!=0: # if there are some samples left in test data that are not enough to form a full batch + batches+=1 # increase the number of batches by 1 to include the remaining samples + index1=batches*batch # calculate the start index of test data for the last batch + index2=batch-(shape0-batches*batch) # calculate how many samples are needed from the beginning of test data to form a full batch + try: + try: + if type(test_data)==list: # if test data is a list of arrays (for multiple if type(test_data)==list: # if test data is a list of arrays (for multiple inputs) + for i in range(len(test_data)): + data_batch[i]=self.platform.concat([test_data[i][index1:],test_data[i][:index2]],0) # concatenate two slices of test data from the end and the beginning to form a batch for each input array + else: # if test data is a single array + data_batch=self.platform.concat([test_data[index1:],test_data[:index2]],0) # concatenate two slices of test data from the end and the beginning to form a batch + if type(test_labels)==list: # if test labels is a list of arrays (for multiple outputs) + for i in range(len(test_labels)): + labels_batch[i]=self.platform.concat([test_labels[i][index1:],test_labels[i][:index2]],0) # concatenate two slices of test labels from the end and the beginning to form a batch for each output array + else: # if test labels is a single array + labels_batch=self.platform.concat([test_labels[index1:],test_labels[:index2]],0) # concatenate two slices of test labels from the end and the beginning to form a batch + except Exception: # if the platform's concat method fails + if type(test_data)==list: # if test data is a list of arrays (for multiple inputs) + for i in range(len(test_data)): + data_batch[i]=np.concatenate([test_data[i][index1:],test_data[i][:index2]],0) # use numpy's concatenate method instead for each input array + else: # if test data is a single array + data_batch=np.concatenate([test_data[index1:],test_data[:index2]],0) # use numpy's concatenate method instead + if type(test_labels)==list: # if test labels is a list of arrays (for multiple outputs) + for i in range(len(test_labels)): + labels_batch[i]=np.concatenate([test_labels[i][index1:],test_labels[i][:index2]],0) # use numpy's concatenate method instead for each output array + else: # if test labels is a single array + labels_batch=np.concatenate([test_labels[index1:],test_labels[:index2]],0) # use numpy's concatenate method instead + except Exception as e: + raise e # raise any other exception + output=self.nn.fp(data_batch) # get the output value using the neural network's forward propagation method + batch_loss=self.nn.loss(output,labels_batch) # get the batch loss value using the neural network's loss function + total_loss+=batch_loss # accumulate the batch loss to total loss + try: + batch_acc=self.nn.accuracy(output,labels_batch) # calculate the batch accuracy using the neural network's accuracy method + total_acc+=batch_acc # accumulate the batch accuracy to total accuracy + except Exception as e: + try: + if self.nn.accuracy!=None: # if the neural network has an accuracy method defined + raise e # raise the exception + except Exception: + pass + test_loss=total_loss.numpy()/batches # convert the total loss value to numpy array and divide it by number of batches to get the average loss value for test data + try: + if self.nn.accuracy!=None: # if the neural network has an accuracy method defined + test_acc=total_acc.numpy()/batches # convert the total accuracy value to numpy array and divide it by number of batches to get the average accuracy value for test data + except Exception: + pass + else: # if batch mode is not used (use all data at once) + output=self.nn.fp(test_data) # get the output value using the neural network's forward propagation method + test_loss=self.nn.loss(output,test_labels) # get the loss value using the neural network's loss function + test_loss=test_loss.numpy() # convert the loss value to numpy array + try: + test_acc=self.nn.accuracy(output,test_labels) # calculate the accuracy value using the neural network's accuracy method + test_acc=test_acc.numpy() # convert the accuracy value to numpy array + except Exception as e: + try: + if self.nn.accuracy!=None: # if the neural network has an accuracy method defined + raise e # raise the exception + except Exception: + pass + try: + if self.nn.accuracy!=None: # if the neural network has an accuracy method defined + return test_loss,test_acc # return the test loss and accuracy values + except Exception: + return test_loss,None # return the test loss value and None + + + def suspend_func(self): # a method to check whether a request to suspend the training or online learning has been made + if self.suspend==True: # if the suspend flag is set to True + if self.save_epoch==None: # if the save epoch number is None, it means no request to save the model has been made + print('Training have suspended.') # print a message to indicate the training or online learning has been suspended + else: + self._save() # save the model using the _save method + while True: # loop indefinitely until resumed by other conditions + if self.suspend==False: # if the suspend flag is set to False, it means a request to resume the training or online learning has been made + print('Training have continued.') # print a message to indicate the training or online learning has been resumed + break # break the loop and resume the training or online learning + return # return nothing for this case + + + def stop_func(self): # a method to check whether the training or online learning has reached the target loss or accuracy values and stop it accordingly + if self.end(): # check whether the target loss or accuracy values have been reached using the end method + self.save(self.total_epoch,True) # save the model using the save method with total epoch number and True flag as inputs + print('\nSystem have stopped training,Neural network have been saved.') # print a message to indicate the training or online learning has been stopped and the model has been saved + self._time=self.time-int(self.time) # get the fractional part of the time elapsed for this training session + if self._time<0.5: # if the fractional part is less than 0.5 + self.time=int(self.time) # round down the time elapsed to integer + else: # if the fractional part is greater than or equal to 0.5 + self.time=int(self.time)+1 # round up the time elapsed to integer + self.total_time+=self.time # calculate and update the total time elapsed for all trainings + print() # print an empty line for readability + print('epoch:{0}'.format(self.total_epoch)) # print the total epoch number for all trainings + if self.test_flag==False: # if test data is not used + print('last loss:{0:.6f}'.format(self.train_loss)) # print the last train loss value + else: # if test data is used + print('last loss:{0:.6f},last test loss:{1:.6f}'.format(self.train_loss,self.test_loss)) # print the last train loss value and test loss value + try: + if self.nn.accuracy!=None: # if the neural network has an accuracy method defined + if self.acc_flag=='%': # if percentage mode is used for accuracy display + if self.test_flag==False: # if test data is not used + print('last accuracy:{0:.1f}'.format(self.train_acc*100)) # print the last train accuracy value in percentage + else: # if test data is used + print('last accuracy:{0:.1f},last test accuracy:{1:.1f}'.format(self.train_acc*100,self.test_acc*100)) # print the last train accuracy value and test accuracy value in percentage + else: # if decimal mode is used for accuracy display + if self.test_flag==False: # if test data is not used + print('last accuracy:{0:.6f}'.format(self.train_acc)) # print the last train accuracy value in decimal + else: # if test data is used + print('last accuracy:{0:.6f},last test accuracy:{1:.6f}'.format(self.train_acc,self.test_acc)) # print the last train accuracy value and test accuracy value in decimal + except Exception: # if the neural network does not have an accuracy method defined or it fails + pass + print() # print an empty line for readability + print('time:{0}s'.format(self.total_time)) # print the total time elapsed for all trainings + self.stop_flag=True # set the stop flag to True, which means the training or online learning has been stopped + return True # return True to indicate that the training or online learning has been stopped + return False # return False to indicate that the training or online learning has not been stopped + + + def stop_func_(self): # a method to check whether a request to stop the training or online learning has been made or the target loss or accuracy values have been reached + if self.stop==True: # if the stop flag is set to True, it means a request to stop the training or online learning has been made + if self.stop_flag==True or self.stop_func(): # if the stop flag is already True, it means the target loss or accuracy values have been reached, or check whether the target loss or accuracy values have been reached using the stop_func method + return True # return True to indicate that the training or online learning has been stopped + return False # return False to indicate that the training or online learning has not been stopped + + + def visualize_train(self): # a method to visualize the train loss and accuracy values using matplotlib.pyplot + print() # print an empty line for readability + plt.figure(1) # create a new figure with number 1 + plt.plot(np.arange(self.total_epoch),self.train_loss_list) # plot the train loss list with x-axis as the epoch number + plt.title('train loss') # set the title of the plot to 'train loss' + plt.xlabel('epoch') # set the x-axis label to 'epoch' + plt.ylabel('loss') # set the y-axis label to 'loss' + print('train loss:{0:.6f}'.format(self.train_loss)) # print the last train loss value + try: + if self.nn.accuracy!=None: # if the neural network has an accuracy method defined + plt.figure(2) # create a new figure with number 2 + plt.plot(np.arange(self.total_epoch),self.train_acc_list) # plot the train accuracy list with x-axis as the epoch number + plt.title('train acc') # set the title of the plot to 'train acc' + plt.xlabel('epoch') # set the x-axis label to 'epoch' + plt.ylabel('acc') # set the y-axis label to 'acc' + if self.acc_flag=='%': # if percentage mode is used for accuracy display + print('train acc:{0:.1f}'.format(self.train_acc*100)) # print the last train accuracy value in percentage + else: # if decimal mode is used for accuracy display + print('train acc:{0:.6f}'.format(self.train_acc)) # print the last train accuracy value in decimal + except Exception: # if the neural network does not have an accuracy method defined or it fails + pass + return # return nothing for this case \ No newline at end of file diff --git a/Note 7.0 documentation/DL/reduced kernel/process/kernel_reduced.py b/Note 7.0 documentation/DL/reduced kernel/process/kernel_reduced.py new file mode 100644 index 00000000..41528280 --- /dev/null +++ b/Note 7.0 documentation/DL/reduced kernel/process/kernel_reduced.py @@ -0,0 +1,637 @@ +import tensorflow as tf +from tensorflow.python.ops import state_ops +from tensorflow.python.util import nest +from multiprocessing import Value,Array +import numpy as np +import matplotlib.pyplot as plt + + +class kernel: + def __init__(self,nn=None): + self.nn=nn # the neural network object + try: + self.nn.km=1 # a flag to indicate the kernel mode + except Exception: + pass + self.PO=None # the order of the optimizer + self.process=None # the number of processes + self.train_ds=None # the training dataset + self.data_segment_flag=False # a flag to indicate whether to segment the data + self.batches=None # the number of batches + self.buffer_size=None # the buffer size for shuffling the data + self.priority_flag=False # a flag to indicate whether to use priority for optimization + self.priority_p=0 # the priority parameter + self.max_opt=None # the maximum number of optimization steps + self.epoch=None # the number of epochs + self.epoch_counter=0 # the epoch counter + self.stop=False # a flag to indicate whether to stop the training + self.stop_flag=False # a flag to indicate whether to stop the training by condition + self.save_flag=False # a flag to indicate whether to save the model + self.batch=None # the batch size + self.end_loss=None # the end condition for training loss + self.end_acc=None # the end condition for training accuracy + self.end_test_loss=None # the end condition for test loss + self.end_test_acc=None # the end condition for test accuracy + self.acc_flag='%' # the format for displaying accuracy + self.opt_counter=None # the counter for optimization steps + self.train_loss=0 # the training loss + self.train_acc=0 # the training accuracy + self.train_loss_list=[] # the list of training loss values + self.train_acc_list=[] # the list of training accuracy values + self.test_loss=0 # the test loss + self.test_acc=0 # the test accuracy + self.test_loss_list=[] # the list of test loss values + self.test_acc_list=[] # the list of test accuracy values + self.test_flag=False # a flag to indicate whether to use test data + self.total_epoch=0 # the total number of epochs + + def data(self,train_data=None,train_labels=None,test_data=None,test_labels=None,train_dataset=None,test_dataset=None): + if type(self.nn.param[0])!=list: + self.train_data=train_data.astype(self.nn.param[0].dtype.name) # convert the train data type to match the model parameter type + self.train_labels=train_labels.astype(self.nn.param[0].dtype.name) # convert the train labels type to match the model parameter type + else: + self.train_data=train_data.astype(self.nn.param[0][0].dtype.name) # convert the train data type to match the model parameter type + self.train_labels=train_labels.astype(self.nn.param[0][0].dtype.name) # convert the train labels type to match the model parameter type + self.train_dataset=train_dataset # set the train dataset object + if test_data is not None: + self.test_data=test_data # set the test data array + self.test_labels=test_labels # set the test labels array + self.test_flag=True # set the test flag to True + self.test_dataset=test_dataset # set the test dataset object + self.batch_counter=np.zeros(self.process,dtype=np.int32) # initialize an array to count batches for each process + if type(self.nn.param[0])!=list: # initialize an array to accumulate loss for each process + self.total_loss=np.zeros(self.process,dtype=self.nn.param[0].dtype.name) + else: + self.total_loss=np.zeros(self.process,dtype=self.nn.param[0][0].dtype.name) + try: + if self.nn.accuracy!=None: + if type(self.nn.param[0])!=list: # initialize an array to accumulate accuracy for each process + self.total_acc=np.zeros(self.process,dtype=self.nn.param[0].dtype.name) + else: + self.total_acc=np.zeros(self.process,dtype=self.nn.param[0][0].dtype.name) + except Exception: + pass + if self.priority_flag==True: + self.opt_counter=np.zeros(self.process,dtype=np.int32) # initialize an array to count optimization steps for each process + if self.train_dataset==None: + if type(self.train_data)==list: + self.shape0=train_data[0].shape[0] # get the number of samples in the first train data array + self.batches=int((self.shape0-self.shape0%self.batch)/self.batch) # calculate the number of batches + if self.shape0%self.batch!=0: + self.batches+=1 # add one more batch if there are remaining samples + else: + self.shape0=train_data.shape[0] # get the number of samples in the train data array + self.batches=int((self.shape0-self.shape0%self.batch)/self.batch) # calculate the number of batches + if self.shape0%self.batch!=0: + self.batches+=1 # add one more batch if there are remaining samples + if self.data_segment_flag==True: + self.train_data,self.train_labels=self.segment_data() # segment the train data and labels according to the number of processes + return + + + def segment_data(self): + if len(self.train_data)!=self.process: # check if the train data is already segmented + segments=int((len(self.train_data)-len(self.train_data)%self.process)/self.process) # calculate the number of samples for each segment + for i in range(self.process): # loop over the processes + index1=i*segments # get the start index of the segment + index2=(i+1)*segments # get the end index of the segment + if i==0: # for the first process + data=np.expand_dims(self.train_data[index1:index2],axis=0) # create a new dimension for the segment and assign it to data + labels=np.expand_dims(self.train_labels[index1:index2],axis=0) # create a new dimension for the segment and assign it to labels + else: # for other processes + data=np.concatenate((data,np.expand_dims(self.train_data[index1:index2],axis=0))) # concatenate the segment to data along the new dimension + labels=np.concatenate((labels,np.expand_dims(self.train_labels[index1:index2],axis=0))) # concatenate the segment to labels along the new dimension + if len(data)%self.process!=0: # check if there are remaining samples that are not segmented + segments+=1 # increase the number of samples for each segment by one + index1=segments*self.process # get the start index of the remaining samples + index2=self.process-(len(self.train_data)-segments*self.process) # get the number of processes that need to be filled with extra samples + data=np.concatenate((data,np.expand_dims(self.train_data[index1:index2],axis=0))) # concatenate the remaining samples to data along the new dimension + labels=np.concatenate((labels,np.expand_dims(self.train_labels[index1:index2],axis=0))) # concatenate the remaining samples to labels along the new dimension + return data,labels # return the segmented data and labels + + + def init(self,manager): + self.epoch_counter=Value('i',self.epoch_counter) # create a shared value for epoch counter + self.batch_counter=Array('i',self.batch_counter) # create a shared array for batch counter + self.total_loss=Array('f',self.total_loss) # create a shared array for total loss + self.total_epoch=Value('i',self.total_epoch) # create a shared value for total epoch + self.train_loss=Value('f',self.train_loss) # create a shared value for train loss + self.train_loss_list=manager.list(self.train_loss_list) # create a shared list for train loss values + self.priority_p=Value('i',self.priority_p) # create a shared value for priority parameter + if self.test_flag==True: + self.test_loss=Value('f',self.test_loss) # create a shared value for test loss + self.test_loss_list=manager.list(self.test_loss_list) # create a shared list for test loss values + try: + if self.nn.accuracy!=None: + self.total_acc=Array('f',self.total_acc) # create a shared array for total accuracy + self.train_acc=Value('f',self.train_acc) # create a shared value for train accuracy + self.train_acc_list=manager.list(self.train_acc_list) # create a shared list for train accuracy values + if self.test_flag==True: + self.test_acc=Value('f',self.test_acc) # create a shared value for test accuracy + self.test_acc_list=manager.list(self.test_acc_list) # create a shared list for test accuracy values + except Exception: + pass + if self.priority_flag==True: + self.opt_counter=Array('i',self.opt_counter) # create a shared array for optimization counter + try: + if self.nn.attenuate!=None: + self.nn.opt_counter=manager.list([self.nn.opt_counter]) # create a shared list for the neural network's optimization counter + except Exception: + pass + try: + self.nn.ec=manager.list([self.nn.ec]) # create a shared list for the neural network's epoch counter + except Exception: + pass + try: + self.nn.bc=manager.list([self.nn.bc]) # create a shared list for the neural network's batch counter + except Exception: + pass + self.stop_flag=Value('b',self.stop_flag) # create a shared value for stop flag + self.save_flag=Value('b',self.save_flag) # create a shared value for save flag + self.param=manager.dict() # create a shared dictionary for parameters + self.param[7]=self.nn.param # assign the neural network's parameters to the dictionary + return + + + def end(self): + if self.end_loss!=None and len(self.train_loss_list)!=0 and self.train_loss_list[-1]self.end_acc: # check if the train accuracy is higher than the end condition + return True + elif self.end_loss!=None and len(self.train_loss_list)!=0 and self.end_acc!=None and self.train_loss_list[-1]self.end_acc: # check if both the train loss and accuracy meet the end condition + return True + elif self.end_test_loss!=None and len(self.test_loss_list)!=0 and self.test_loss_list[-1]self.end_test_acc: # check if the test accuracy is higher than the end condition + return True + elif self.end_test_loss!=None and len(self.test_loss_list)!=0 and self.end_test_acc!=None and self.test_loss_list[-1]self.end_test_acc: # check if both the test loss and accuracy meet the end condition + return True + + + @tf.function + def opt_p(self,data,labels,p,lock,g_lock=None): + try: + try: + if self.nn.GradientTape!=None: # check if the neural network has its own gradient tape function + tape,output,loss=self.nn.GradientTape(data,labels,p) # use the neural network's gradient tape function to get the tape, output and loss + except Exception: + with tf.GradientTape(persistent=True) as tape: # use the default gradient tape function + try: + try: + output=self.nn.fp(data) # use the neural network's forward propagation function to get the output + loss=self.nn.loss(output,labels) # use the neural network's loss function to get the loss + except Exception: + output,loss=self.nn.fp(data,labels) # use the neural network's forward propagation function to get the output and loss + except Exception: + try: + output=self.nn.fp(data,p) # use the neural network's forward propagation function to get the output with the process number + loss=self.nn.loss(output,labels) # use the neural network's loss function to get the loss + except Exception: + output,loss=self.nn.fp(data,labels,p) # use the neural network's forward propagation function to get the output and loss with the process number + except Exception as e: + raise e + if self.PO==1: # check if the optimizer order is 1 (lock before gradient calculation) + if self.priority_flag==True and self.priority_p.value!=-1: # check if the priority flag is True and the priority parameter is not -1 + while True: + if p==self.priority_p.value: # check if the process number matches the priority parameter + break + else: + continue + lock[0].acquire() # acquire the first lock + if self.stop_func_(lock[0]): # check if the stop condition is met + return None,0 + try: + try: + try: + gradient=self.nn.gradient(tape,loss) # use the neural network's gradient function to get the gradient + except Exception: + gradient=self.nn.gradient(tape,loss,self.param[7]) # use the neural network's gradient function to get the gradient with the parameters + except Exception: + gradient=tape.gradient(loss,self.nn.param) # use the default gradient function to get the gradient + except Exception as e: + raise e + try: + gradient=self.nn.attenuate(gradient,self.nn.opt_counter,p) # use the neural network's attenuate function to modify the gradient + except Exception as e: + try: + if self.nn.attenuate!=None: # check if the neural network has an attenuate function + raise e + except Exception: + pass + try: + try: + param=self.nn.opt(gradient) # use the neural network's optimizer function to update the parameters + except Exception: + param=self.nn.opt(gradient,p) # use the neural network's optimizer function to update the parameters with the process number + except Exception as e: + raise e + lock[0].release() # release the first lock + elif self.PO==2: # check if the optimizer order is 2 (lock after gradient calculation) + g_lock.acquire() # acquire the global lock + if self.stop_func_(g_lock): # check if the stop condition is met + return None,0 + try: + try: + try: + gradient=self.nn.gradient(tape,loss) # use the neural network's gradient function to get the gradient + except Exception: + gradient=self.nn.gradient(tape,loss,self.param[7]) # use the neural network's gradient function to get the gradient with the parameters + except Exception: + gradient=tape.gradient(loss,self.nn.param) # use the default gradient function to get the gradient + except Exception as e: + raise e + g_lock.release() # release the global lock + if self.priority_flag==True and self.priority_p.value!=-1: # check if the priority flag is True and the priority parameter is not -1 + while True: + if p==self.priority_p.value: # check if the process number matches the priority parameter + break + else: + continue + lock[0].acquire() # acquire the first lock + if self.stop_func_(lock[0]): # check if the stop condition is met + return None,0 + try: + gradient=self.nn.attenuate(gradient,self.nn.opt_counter,p) # use the neural network's attenuate function to modify the gradient + except Exception as e: + try: + if self.nn.attenuate!=None: # check if the neural network has an attenuate function + raise e + except Exception: + pass + try: + try: + param=self.nn.opt(gradient) # use the neural network's optimizer function to update the parameters + except Exception: + param=self.nn.opt(gradient,p) # use the neural network's optimizer function to update the parameters with the process number + except Exception as e: + raise e + lock[0].release() # release the first lock + elif self.PO==3: # check if the optimizer order is 3 (no lock) + if self.priority_flag==True and self.priority_p.value!=-1: # check if the priority flag is True and the priority parameter is not -1 + while True: + if p==self.priority_p.value: # check if the process number matches the priority parameter + break + else: + continue + if self.stop_func_(): # check if the stop condition is met + return None,0 + try: + try: + try: + gradient=self.nn.gradient(tape,loss) # use the neural network's gradient function to get the gradient + except Exception: + gradient=self.nn.gradient(tape,loss,self.param[7]) # use the neural network's gradient function to get the gradient with the parameters + except Exception: + gradient=tape.gradient(loss,self.nn.param) # use the default gradient function to get the gradient + except Exception as e: + raise e + try: + gradient=self.nn.attenuate(gradient,self.nn.opt_counter,p) # use the neural network's attenuate function to modify the gradient + except Exception as e: + try: + if self.nn.attenuate!=None: # check if the neural network has an attenuate function + raise e + except Exception: + pass + try: + try: + param=self.nn.opt(gradient) # use the neural network's optimizer function to update the parameters + except Exception: + param=self.nn.opt(gradient,p) # use the neural network's optimizer function to update the parameters with the process number + except Exception as e: + raise e + return output,loss,param # return output, loss and parameters + + + def opt(self,data,labels,p,lock,g_lock): + if self.PO==2: # check if the optimizer order is 2 (lock after gradient calculation) + if type(g_lock)!=list: # check if g_lock is not a list + pass + elif len(g_lock)==self.process: # check if g_lock has same length as process number + ln=p # assign process number to ln (local number) + g_lock=g_lock[ln] # assign g_lock element at ln index to g_lock + else: + ln=int(np.random.choice(len(g_lock))) # assign a random integer from g_lock length to ln (local number) + g_lock=g_lock[ln] # assign g_lock element at ln index to g_lock + output,loss,param=self.opt_p(data,labels,p,lock,g_lock) # call the opt_p function to get output, loss and parameters + else: + output,loss,param=self.opt_p(data,labels,p,lock) # call the opt_p function to get output, loss and parameters without g_lock + return output,loss,param # return output, loss and parameters + + + def update_nn_param(self,param=None): + if param==None: # check if param is None + parameter_flat=nest.flatten(self.nn.param) # flatten the neural network's parameters + parameter7_flat=nest.flatten(self.param[7]) # flatten the kernel's parameters + else: + parameter_flat=nest.flatten(self.nn.param) # flatten the neural network's parameters + parameter7_flat=nest.flatten(param) # flatten the given param + for i in range(len(parameter_flat)): # loop over the flattened parameters + if param==None: # check if param is None + state_ops.assign(parameter_flat[i],parameter7_flat[i]) # assign the kernel's parameters to the neural network's parameters + else: + state_ops.assign(parameter_flat[i],parameter7_flat[i]) # assign the given param to the neural network's parameters + self.nn.param=nest.pack_sequence_as(self.nn.param,parameter_flat) # pack the flattened parameters back to the neural network's parameters + self.param[7]=nest.pack_sequence_as(self.param[7],parameter7_flat) # pack the flattened parameters back to the kernel's parameters + return + + + def train7(self,train_ds,p,test_batch,lock,g_lock): + while True: # loop until break + for data_batch,labels_batch in train_ds: # loop over the train dataset batches + try: + data_batch,labels_batch=self.nn.data_func(data_batch,labels_batch) # use the neural network's data function to process data and labels + except Exception as e: + try: + if self.nn.data_func!=None: # check if the neural network has a data function + raise e + except Exception: + pass + if self.priority_flag==True: # check if the priority flag is True + self.priority_p.value=np.argmax(self.opt_counter) # assign the index of the maximum value in opt_counter to priority parameter + if self.max_opt!=None and self.opt_counter[self.priority_p.value]>=self.max_opt: # check if max_opt is not None and opt_counter at priority parameter index is greater than or equal to max_opt + self.priority_p.value=int(self.priority_p.value) # convert priority parameter to integer type + elif self.max_opt==None: # check if max_opt is None + self.priority_p.value=int(self.priority_p.value) # convert priority parameter to integer type + else: + self.priority_p.value=-1 # assign -1 to priority parameter + if self.priority_flag==True: # check if the priority flag is True + self.opt_counter[p]=0 # assign zero to opt_counter at process number + try: + opt_counter=self.nn.opt_counter[0] # get the neural network's optimization counter + opt_counter.scatter_update(tf.IndexedSlices(0,p)) # update opt_counter with zero at process number + self.nn.opt_counter[0]=opt_counter # assign opt_counter back to neural network's optimization counter + except Exception as e: + try: + if self.nn.attenuate!=None: # check if the neural network has an attenuate function + raise e + except Exception: + pass + output,batch_loss,param=self.opt(data_batch,labels_batch,p,lock,g_lock) # call the opt function to get output, batch loss and parameters + self.param[7]=param # assign param to kernel's parameters + if self.priority_flag==True: # check if the priority flag is True + opt_counter=np.frombuffer(self.opt_counter.get_obj(),dtype='i') # get a numpy array from opt_counter shared array object + opt_counter+=1 # increment opt_counter by one for each element + try: + opt_counter=self.nn.opt_counter[0] # get the neural network's optimization counter + opt_counter.assign(opt_counter+1) # increment opt_counter by one + self.nn.opt_counter[0]=opt_counter # assign opt_counter back to neural network's optimization counter + except Exception as e: + try: + if self.nn.attenuate!=None: # check if the neural network has an attenuate function + raise e + except Exception: + pass + try: + bc=self.nn.bc[0] # get the neural network's batch counter + bc.assign_add(1) # increment bc by one + self.nn.bc[0]=bc # assign bc back to neural network's batch counter + except Exception: + pass + try: + batch_acc=self.nn.accuracy(output,labels_batch) # use the neural network's accuracy function to get the batch accuracy + except Exception as e: + try: + if self.nn.accuracy!=None: # check if the neural network has an accuracy function + raise e + except Exception: + pass + try: + if self.nn.accuracy!=None: # check if the neural network has an accuracy function + self.total_loss[p]+=batch_loss # accumulate batch loss to total loss at process number + self.total_acc[p]+=batch_acc # accumulate batch accuracy to total accuracy at process number + except Exception: + self.total_loss[p]+=batch_loss # accumulate batch loss to total loss at process number + self.batch_counter[p]+=1 # increment batch counter at process number + if self.PO==1 or self.PO==2: # check if the optimizer order is 1 or 2 (lock before or after gradient calculation) + lock[1].acquire() # acquire the second lock + elif lock!=None: # check if lock is not None + lock.acquire() # acquire the lock + batches=np.sum(self.batch_counter) # sum up the batch counter for all processes + if batches>=self.batches: # check if the number of batches reaches the total number of batches + batch_counter=np.frombuffer(self.batch_counter.get_obj(),dtype='i') # get a numpy array from batch counter shared array object + batch_counter*=0 # reset batch counter to zero for each element + loss=np.sum(self.total_loss)/batches # calculate the average loss for all batches + try: + if self.nn.accuracy!=None: # check if the neural network has an accuracy function + train_acc=np.sum(self.total_acc)/batches # calculate the average accuracy for all batches + except Exception: + pass + self.total_epoch.value+=1 # increment total epoch by one + self.train_loss.value=loss # assign loss to train loss + self.train_loss_list.append(loss) # append loss to train loss list + try: + if self.nn.accuracy!=None: # check if the neural network has an accuracy function + self.train_acc.value=train_acc # assign train_acc to train accuracy + self.train_acc_list.append(train_acc) # append train_acc to train accuracy list + except Exception: + pass + if self.test_flag==True: # check if the test flag is True + self.test_loss.value,self.test_acc.value=self.test(self.test_data,self.test_labels,test_batch,p) # call the test function to get test loss and accuracy + self.test_loss_list.append(self.test_loss.value) # append test loss to test loss list + try: + if self.nn.accuracy!=None: # check if the neural network has an accuracy function + self.test_acc_list.append(self.test_acc.value) # append test accuracy to test accuracy list + except Exception: + pass + self.print_save() # call the print_save function to print and save the results + self.epoch_counter.value+=1 # increment epoch counter by one + try: + ec=self.nn.ec[0] # get the neural network's epoch counter + ec.assign_add(1) # increment ec by one + self.nn.ec[0]=ec # assign ec back to neural network's epoch counter + except Exception: + pass + total_loss=np.frombuffer(self.total_loss.get_obj(),dtype='f') # get a numpy array from total loss shared array object + total_loss*=0 # reset total loss to zero for each element + try: + total_acc=np.frombuffer(self.total_acc.get_obj(),dtype='f') # get a numpy array from total accuracy shared array object + total_acc*=0 # reset total accuracy to zero for each element + except Exception as e: + try: + if self.nn.accuracy!=None: # check if the neural network has an accuracy function + raise e + except Exception: + pass + if self.PO==1 or self.PO==2: # check if the optimizer order is 1 or 2 (lock before or after gradient calculation) + lock[1].release() # release the second lock + elif lock!=None: # check if lock is not None + lock.release() # release the lock + if self.epoch_counter.value>=self.epoch: # check if the epoch counter reaches the epoch number + self.param[7]=param # assign param to kernel's parameters + return + + + def train(self,p,lock=None,g_lock=None,test_batch=None): + if self.epoch!=None: # check if epoch is not None + if self.train_dataset!=None: # check if train dataset is not None + train_ds=self.train_dataset # assign train dataset to train_ds + else: + if self.data_segment_flag==True: # check if data segment flag is True + train_ds=tf.data.Dataset.from_tensor_slices((self.train_data[p],self.train_labels[p])).batch(self.batch) # create a dataset from tensor slices of segmented data and labels and batch them + elif self.buffer_size!=None: # check if buffer size is not None + train_ds=tf.data.Dataset.from_tensor_slices((self.train_data,self.train_labels)).shuffle(self.buffer_size).batch(self.batch) # create a dataset from tensor slices of data and labels and shuffle and batch them with buffer size + else: + train_ds=tf.data.Dataset.from_tensor_slices((self.train_data,self.train_labels)).batch(self.batch) # create a dataset from tensor slices of data and labels and batch them + self.train7(train_ds,p,test_batch,lock,g_lock) # call the train7 function with train_ds, process number, test batch, lock and g_lock + return + + + def test(self,test_data=None,test_labels=None,batch=None,p=None): + if type(self.nn.param[0])!=list: + test_data=test_data.astype(self.nn.param[0].dtype.name) # convert the test data type to match the model parameter type + test_labels=test_labels.astype(self.nn.param[0].dtype.name) # convert the test labels type to match the model parameter type + else: + test_data=test_data.astype(self.nn.param[0][0].dtype.name) # convert the test data type to match the model parameter type + test_labels=test_labels.astype(self.nn.param[0][0].dtype.name) # convert the test labels type to match the model parameter type + if batch!=None: # check if batch is not None + total_loss=0 # initialize total loss to zero + total_acc=0 # initialize total accuracy to zero + if self.test_dataset!=None: # check if test dataset is not None + for data_batch,labels_batch in self.test_dataset: # loop over the test dataset batches + try: + try: + output=self.nn.fp(data_batch) # use the neural network's forward propagation function to get the output + except Exception: + output=self.nn.fp(data_batch,p) # use the neural network's forward propagation function to get the output with the process number + except Exception as e: + raise e + batch_loss=self.nn.loss(output,labels_batch) # use the neural network's loss function to get the batch loss + total_loss+=batch_loss # accumulate batch loss to total loss + try: + batch_acc=self.nn.accuracy(output,labels_batch) # use the neural network's accuracy function to get the batch accuracy + total_acc+=batch_acc # accumulate batch accuracy to total accuracy + except Exception as e: + try: + if self.nn.accuracy!=None: # check if the neural network has an accuracy function + raise e + except Exception: + pass + else: # if test dataset is None + total_loss=0 # initialize total loss to zero + total_acc=0 # initialize total accuracy to zero + batches=int((test_data.shape[0]-test_data.shape[0]%batch)/batch) # calculate the number of batches + shape0=test_data.shape[0] # get the number of samples in the test data array + for j in range(batches): # loop over the batches + index1=j*batch # get the start index of the batch + index2=(j+1)*batch # get the end index of the batch + data_batch=test_data[index1:index2] # get the data batch from test data array + if type(test_labels)==list: # check if test labels is a list + for i in range(len(test_labels)): + labels_batch[i]=test_labels[i][index1:index2] # get the labels batch from test labels list + else: + labels_batch=test_labels[index1:index2] # get the labels batch from test labels array + try: + try: + output=self.nn.fp(data_batch) # use the neural network's forward propagation function to get the output + except Exception: + output=self.nn.fp(data_batch,p) # use the neural network's forward propagation function to get the output with the process number + except Exception as e: + raise e + batch_loss=self.nn.loss(output,labels_batch) # use the neural network's loss function to get the batch loss + total_loss+=batch_loss # accumulate batch loss to total loss + try: + batch_acc=self.nn.accuracy(output,labels_batch) # use the neural network's accuracy function to get the batch accuracy + total_acc+=batch_acc # accumulate batch accuracy to total accuracy + except Exception as e: + try: + if self.nn.accuracy!=None: # check if the neural network has an accuracy function + raise e + except Exception: + pass + if shape0%batch!=0: # check if there are remaining samples that are not in a batch + batches+=1 # increment batches by one + index1=batches*batch # get the start index of the remaining samples + index2=batch-(shape0-batches*batch) # get the number of samples that need to be filled with extra samples + data_batch=tf.concat([test_data[index1:],test_data[:index2]],0) # concatenate the remaining samples and extra samples to form a data batch + labels_batch=tf.concat([test_labels[index1:],test_labels[:index2]],0) # concatenate the remaining labels and extra labels to form a labels batch + try: + try: + output=self.nn.fp(data_batch) # use the neural network's forward propagation function to get the output + except Exception: + output=self.nn.fp(data_batch,p) # use the neural network's forward propagation function to get the output with the process number + except Exception as e: + raise e + batch_loss=self.nn.loss(output,labels_batch) # use the neural network's loss function to get the batch loss + total_loss+=batch_loss # accumulate batch loss to total loss + try: + batch_acc=self.nn.accuracy(output,labels_batch) # use the neural network's accuracy function to get the batch accuracy + total_acc+=batch_acc # accumulate batch accuracy to total accuracy + except Exception as e: + try: + if self.nn.accuracy!=None: # check if the neural network has an accuracy function + raise e + except Exception: + pass + test_loss=total_loss.numpy()/batches # calculate the average test loss for all batches + try: + if self.nn.accuracy!=None: # check if the neural network has an accuracy function + test_acc=total_acc.numpy()/batches # calculate the average test accuracy for all batches + except Exception: + pass + else: # if batch is None + try: + try: + output=self.nn.fp(test_data) # use the neural network's forward propagation function to get the output + except Exception: + output=self.nn.fp(test_data,p) # use the neural network's forward propagation function to get the output with the process number + except Exception as e: + raise e + test_loss=self.nn.loss(output,test_labels) # use the neural network's loss function to get the test loss + test_loss=test_loss.numpy() # convert test loss to numpy array + try: + test_acc=self.nn.accuracy(output,test_labels) # use the neural network's accuracy function to get the test accuracy + test_acc=test_acc.numpy() # convert test accuracy to numpy array + except Exception as e: + try: + if self.nn.accuracy!=None: # check if the neural network has an accuracy function + raise e + except Exception: + pass + try: + if self.nn.accuracy!=None: # check if the neural network has an accuracy function + return test_loss,test_acc # return test loss and accuracy + except Exception: + return test_loss,None # return test loss and None + + + def stop_func(self): + if self.end(): # check if any of the end conditions is met + self.save(self.total_epoch.value,True) # save the model with total epoch and True flag + self.save_flag.value=True # set save flag to True + self.stop_flag.value=True # set stop flag to True + return True + return False + + + def stop_func_(self,lock=None): + if self.stop==True: # check if stop is True + if self.stop_flag.value==True or self.stop_func(): # check if the stop flag is True or the stop function returns True + if self.PO!=3: # check if the optimizer order is not 3 (no lock) + lock.release() # release the lock + return True + return False + + + def visualize_train(self): + print() # print a blank line + plt.figure(1) # create a new figure + plt.plot(np.arange(self.total_epoch.value),self.train_loss_list) # plot the train loss list against the total epoch + plt.title('train loss') # set the title of the figure + plt.xlabel('epoch') # set the x-axis label of the figure + plt.ylabel('loss') # set the y-axis label of the figure + print('train loss:{0:.6f}'.format(self.train_loss.value)) # print the train loss value with six decimal places + try: + if self.nn.accuracy!=None: # check if the neural network has an accuracy function + plt.figure(2) # create a new figure + plt.plot(np.arange(self.total_epoch.value),self.train_acc_list) # plot the train accuracy list against the total epoch + plt.title('train acc') # set the title of the figure + plt.xlabel('epoch') # set the x-axis label of the figure + plt.ylabel('acc') # set the y-axis label of the figure + if self.acc_flag=='%': # check if the accuracy format is percentage + print('train acc:{0:.1f}'.format(self.train_acc.value*100)) # print the train accuracy value with one decimal place and percentage sign + else: + print('train acc:{0:.6f}'.format(self.train_acc.value)) # print the train accuracy value with six decimal places + except Exception: + pass + return \ No newline at end of file diff --git a/Note 7.0 documentation/RL/kernel.txt b/Note 7.0 documentation/RL/kernel.txt new file mode 100644 index 00000000..321998a2 --- /dev/null +++ b/Note 7.0 documentation/RL/kernel.txt @@ -0,0 +1,10 @@ +Pool network: +Pool network use multiprocessing parallel and random add episode in pool,which would make data being uncorrelated in pool, +then pools would be used parallel training agent. +Each process have a pool. + +Support stop mechanism. +Support prioritized replay. +Support gradient attenuation. +Support memory protection mechanism. +Support training priority and memory priority. \ No newline at end of file diff --git a/Note 7.0 documentation/RL/neural network/pytorch/DDPG.py b/Note 7.0 documentation/RL/neural network/pytorch/DDPG.py new file mode 100644 index 00000000..4b0abd94 --- /dev/null +++ b/Note 7.0 documentation/RL/neural network/pytorch/DDPG.py @@ -0,0 +1,101 @@ +import torch +import torch.nn.functional as F +import numpy as np +import gym + + +class actor(torch.nn.Module): + def __init__(self,state_dim,hidden_dim,action_dim,action_bound): + super(actor,self).__init__() + self.fc1=torch.nn.Linear(state_dim,hidden_dim) + self.fc2=torch.nn.Linear(hidden_dim,action_dim) + self.action_bound=action_bound + + + def forward(self,x): + x=F.relu(self.fc1(x)) + return torch.tanh(self.fc2(x))*self.action_bound + + +class critic(torch.nn.Module): + def __init__(self,state_dim,hidden_dim,action_dim): + super(critic,self).__init__() + self.fc1=torch.nn.Linear(state_dim+action_dim,hidden_dim) + self.fc2=torch.nn.Linear(hidden_dim,1) + + + def forward(self,x,a): + cat=torch.cat([x,a],dim=1) + x=F.relu(self.fc1(cat)) + return self.fc2(x) + + +class DDPG: + def __init__(self,hidden_dim,sigma,gamma,tau,actor_lr,critic_lr): + if torch.cuda.is_available(): + self.device=torch.device('cuda') + else: + self.device=torch.device('cpu') + self.genv=gym.make('Pendulum-v0') + state_dim=self.genv.observation_space.shape[0] + action_dim=self.genv.action_space.shape[0] + action_bound=self.genv.action_space.high[0] + self.actor=actor(state_dim,hidden_dim,action_dim,action_bound).to(self.device) + self.critic=critic(state_dim,hidden_dim,action_dim).to(self.device) + self.target_actor=actor(state_dim,hidden_dim,action_dim,action_bound).to(self.device) + self.target_critic=critic(state_dim,hidden_dim,action_dim).to(self.device) + self.target_actor.load_state_dict(self.actor.state_dict()) + self.target_critic.load_state_dict(self.critic.state_dict()) + self.sigma=sigma + self.gamma=gamma + self.tau=tau + self.actor_opt=torch.optim.Adam(self.actor.parameters(),lr=actor_lr) + self.critic_opt=torch.optim.Adam(self.critic.parameters(),lr=critic_lr) + + + def noise(self): + return np.random.normal(scale=self.sigma) + + + def env(self,a=None,initial=None): + if initial==True: + state=self.genv.reset(seed=0) + return state + else: + next_state,reward,done,_=self.genv.step(a) + return next_state,reward,done + + + def loss(self,s,a,next_s,r,d): + s=torch.tensor(s,dtype=torch.float).to(self.device) + a=torch.tensor(a,dtype=torch.float).view(-1,1).to(self.device) + next_s=torch.tensor(next_s,dtype=torch.float).to(self.device) + r=torch.tensor(r,dtype=torch.float).view(-1,1).to(self.device) + d=torch.tensor(d,dtype=torch.float).view(-1,1).to(self.device) + next_q_value=self.target_critic(next_s,self.target_actor(next_s)) + q_target=r+self.gamma*next_q_value*(1-d) + actor_loss=-torch.mean(self.critic(s,self.actor(s))) + critic_loss=F.mse_loss(self.critic(s,a),q_target) + return [actor_loss,critic_loss] + + + def backward(self,loss): + self.actor_opt.zero_grad() + loss[0].backward() + self.critic_opt.zero_grad() + loss[1].backward() + return + + + def opt(self): + self.actor_opt.step() + self.critic_opt.step() + return + + + def update_param(self): + for target_param,param in zip(self.target_actor.parameters(),self.actor.parameters()): + target_param.data.copy_(target_param.data*(1.0-self.tau)+param.data*self.tau) + for target_param,param in zip(self.target_critic.parameters(),self.critic.parameters()): + target_param.data.copy_(target_param.data*(1.0-self.tau)+param.data*self.tau) + return \ No newline at end of file diff --git a/Note 7.0 documentation/RL/neural network/pytorch/DQN.py b/Note 7.0 documentation/RL/neural network/pytorch/DQN.py new file mode 100644 index 00000000..a0e58689 --- /dev/null +++ b/Note 7.0 documentation/RL/neural network/pytorch/DQN.py @@ -0,0 +1,64 @@ +import torch +import gym +import torch.nn.functional as F + + +class Qnet(torch.nn.Module): + def __init__(self,state_dim,hidden_dim,action_dim): + super(Qnet,self).__init__() + self.fc1=torch.nn.Linear(state_dim,hidden_dim) + self.fc2=torch.nn.Linear(hidden_dim,action_dim) + + + def forward(self,x): + x=F.relu(self.fc1(x)) + return self.fc2(x) + + +class DQN: + def __init__(self,state_dim,hidden_dim,action_dim): + if torch.cuda.is_available(): + self.device=torch.device('cuda') + else: + self.device=torch.device('cpu') + self.nn=Qnet(state_dim,hidden_dim,action_dim).to(self.device) + self.target_q_net=Qnet(state_dim,hidden_dim,action_dim).to(self.device) + self.optimizer=torch.optim.Adam(self.nn.parameters(),lr=2e-3) #optimizer,kernel uses it to optimize. + self.genv=gym.make('CartPole-v0') #create environment + + + def env(self,a=None,initial=None): #environment function,kernel uses it to interact with the environment. + if initial==True: + state=self.genv.reset() + return state + else: + next_state,reward,done,_=self.genv.step(a) + return next_state,reward,done + + + def loss(self,s,a,next_s,r,d): #loss function,kernel uses it to calculate loss. + s=torch.tensor(s,dtype=torch.float).to(self.device) + a=torch.tensor(a,dtype=torch.int64).view(-1,1).to(self.device) + next_s=torch.tensor(next_s,dtype=torch.float).to(self.device) + r=torch.tensor(r,dtype=torch.float).view(-1,1).to(self.device) + d=torch.tensor(d,dtype=torch.float).view(-1,1).to(self.device) + q_value=self.nn(s).gather(1,a) + next_q_value=self.target_q_net(next_s).max(1)[0].view(-1,1) + target=r+0.98*next_q_value*(1-d) + return F.mse_loss(q_value,target) + + + def backward(self,loss): #backward function,kernel uses it for backpropagation. + self.optimizer.zero_grad() + loss.backward() + return + + + def opt(self): #opt function,kernel uses it to optimize. + self.optimizer.step() + return + + + def update_param(self): #update function,kernel uses it to update parameter. + self.target_q_net.load_state_dict(self.nn.state_dict()) + return \ No newline at end of file diff --git a/Note 7.0 documentation/RL/neural network/pytorch/DQN_pr.py b/Note 7.0 documentation/RL/neural network/pytorch/DQN_pr.py new file mode 100644 index 00000000..66c71847 --- /dev/null +++ b/Note 7.0 documentation/RL/neural network/pytorch/DQN_pr.py @@ -0,0 +1,76 @@ +import torch +import gym +import torch.nn.functional as F +import Note.create.RL.rl.prioritized_replay as pr + +#prioritized replay example +class Qnet(torch.nn.Module): + def __init__(self,state_dim,hidden_dim,action_dim): + super(Qnet,self).__init__() + self.fc1=torch.nn.Linear(state_dim,hidden_dim) + self.fc2=torch.nn.Linear(hidden_dim,action_dim) + + + def forward(self,x): + x=F.relu(self.fc1(x)) + return self.fc2(x) + + +class DQN: + def __init__(self,state_dim,hidden_dim,action_dim): + if torch.cuda.is_available(): + self.device=torch.device('cuda') + else: + self.device=torch.device('cpu') + self.nn=Qnet(state_dim,hidden_dim,action_dim).to(self.device) + self.target_q_net=Qnet(state_dim,hidden_dim,action_dim).to(self.device) + self.pr=pr.pr() + self.initial_TD=7 + self._epsilon=0.0007 + self.alpha=0.7 + self.optimizer=torch.optim.Adam(self.nn.parameters(),lr=2e-3) + self.genv=gym.make('CartPole-v0') + + + def env(self,a=None,initial=None): + if initial==True: + state=self.genv.reset(seed=0) + return state + else: + next_state,reward,done,_=self.genv.step(a) + return next_state,reward,done + + + def data_func(self,state_pool,action_pool,next_state_pool,reward_pool,done_pool,batch): + s,a,next_s,r,d=self.pr.sample(state_pool,action_pool,next_state_pool,reward_pool,done_pool,self.epsilon,self.alpha,batch) + return s,a,next_s,r,d + + + def loss(self,s,a,next_s,r,d): + s=torch.tensor(s,dtype=torch.float).to(self.device) + a=torch.tensor(a,dtype=torch.int64).view(-1,1).to(self.device) + next_s=torch.tensor(next_s,dtype=torch.float).to(self.device) + r=torch.tensor(r,dtype=torch.float).view(-1,1).to(self.device) + d=torch.tensor(d,dtype=torch.float).view(-1,1).to(self.device) + q_value=self.nn(s).gather(1,a) + next_q_value=self.target_q_net(next_s).max(1)[0].view(-1,1) + target=r+0.98*next_q_value*(1-d) + TD=target-q_value + self.pr.update_TD(TD) + return torch.mean(TD**2) + + + def backward(self,loss): + self.optimizer.zero_grad() + loss.backward() + return + + + def opt(self): + self.optimizer.step() + return + + + def update_param(self): + self.target_q_net.load_state_dict(self.nn.state_dict()) + return \ No newline at end of file diff --git a/Note 7.0 documentation/RL/neural network/pytorch/DoubleDQN.py b/Note 7.0 documentation/RL/neural network/pytorch/DoubleDQN.py new file mode 100644 index 00000000..31e606e8 --- /dev/null +++ b/Note 7.0 documentation/RL/neural network/pytorch/DoubleDQN.py @@ -0,0 +1,65 @@ +import torch +import gym +import torch.nn.functional as F + + +class Qnet(torch.nn.Module): + def __init__(self,state_dim,hidden_dim,action_dim): + super(Qnet,self).__init__() + self.fc1=torch.nn.Linear(state_dim,hidden_dim) + self.fc2=torch.nn.Linear(hidden_dim,action_dim) + + + def forward(self,x): + x=F.relu(self.fc1(x)) + return self.fc2(x) + + +class DoubleDQN: + def __init__(self,state_dim,hidden_dim,action_dim): + if torch.cuda.is_available(): + self.device=torch.device('cuda') + else: + self.device=torch.device('cpu') + self.nn=Qnet(state_dim,hidden_dim,action_dim).to(self.device) + self.target_q_net=Qnet(state_dim,hidden_dim,action_dim).to(self.device) + self.optimizer=torch.optim.Adam(self.nn.parameters(),lr=2e-3) + self.genv=gym.make('CartPole-v0') + + + def env(self,a=None,initial=None): + if initial==True: + state=self.genv.reset(seed=0) + return state + else: + next_state,reward,done,_=self.genv.step(a) + return next_state,reward,done + + + def loss(self,s,a,next_s,r,d): + s=torch.tensor(s,dtype=torch.float).to(self.device) + a=torch.tensor(a,dtype=torch.int64).view(-1,1).to(self.device) + next_s=torch.tensor(next_s,dtype=torch.float).to(self.device) + r=torch.tensor(r,dtype=torch.float).view(-1,1).to(self.device) + d=torch.tensor(d,dtype=torch.float).view(-1,1).to(self.device) + q_value=self.nn(s).gather(1,a) + max_action=self.nn(next_s).max(1)[1].view(-1,1) + next_q_value=self.target_q_net(next_s).gather(1,max_action) + target=r+0.98*next_q_value*(1-d) + return F.mse_loss(q_value,target) + + + def backward(self,loss): + self.optimizer.zero_grad() + loss.backward() + return + + + def opt(self): + self.optimizer.step() + return + + + def update_param(self): + self.target_q_net.load_state_dict(self.nn.state_dict()) + return \ No newline at end of file diff --git a/Note 7.0 documentation/RL/neural network/pytorch/DuelingDQN.py b/Note 7.0 documentation/RL/neural network/pytorch/DuelingDQN.py new file mode 100644 index 00000000..b81eeb37 --- /dev/null +++ b/Note 7.0 documentation/RL/neural network/pytorch/DuelingDQN.py @@ -0,0 +1,67 @@ +import torch +import gym +import torch.nn.functional as F + + +class VAnet(torch.nn.Module): + def __init__(self,state_dim,hidden_dim,action_dim): + super(VAnet,self).__init__() + self.fc1=torch.nn.Linear(state_dim,hidden_dim) + self.fc_A=torch.nn.Linear(hidden_dim,action_dim) + self.fc_V=torch.nn.Linear(hidden_dim,1) + + + def forward(self,x): + A=self.fc_A(F.relu(self.fc1(x))) + V=self.fc_V(F.relu(self.fc1(x))) + Q=V+A-A.mean(1).view(-1,1) + return Q + + +class DuelingDQN: + def __init__(self,state_dim,hidden_dim,action_dim): + if torch.cuda.is_available(): + self.device=torch.device('cuda') + else: + self.device=torch.device('cpu') + self.nn=VAnet(state_dim,hidden_dim,action_dim).to(self.device) + self.target_q_net=VAnet(state_dim,hidden_dim,action_dim).to(self.device) + self.optimizer=torch.optim.Adam(self.nn.parameters(),lr=2e-3) + self.genv=gym.make('CartPole-v0') + + + def env(self,a=None,initial=None): + if initial==True: + state=self.genv.reset(seed=0) + return state + else: + next_state,reward,done,_=self.genv.step(a) + return next_state,reward,done + + + def loss(self,s,a,next_s,r,d): + s=torch.tensor(s,dtype=torch.float).to(self.device) + a=torch.tensor(a,dtype=torch.int64).view(-1,1).to(self.device) + next_s=torch.tensor(next_s,dtype=torch.float).to(self.device) + r=torch.tensor(r,dtype=torch.float).view(-1,1).to(self.device) + d=torch.tensor(d,dtype=torch.float).view(-1,1).to(self.device) + q_value=self.nn(s).gather(1,a) + next_q_value=self.target_q_net(next_s).max(1)[0].view(-1,1) + target=r+0.98*next_q_value*(1-d) + return F.mse_loss(q_value,target) + + + def backward(self,loss): + self.optimizer.zero_grad() + loss.backward() + return + + + def opt(self): + self.optimizer.step() + return + + + def update_param(self): + self.target_q_net.load_state_dict(self.nn.state_dict()) + return \ No newline at end of file diff --git a/Note 7.0 documentation/RL/neural network/tensorflow/DDPG.py b/Note 7.0 documentation/RL/neural network/tensorflow/DDPG.py new file mode 100644 index 00000000..3bd32083 --- /dev/null +++ b/Note 7.0 documentation/RL/neural network/tensorflow/DDPG.py @@ -0,0 +1,84 @@ +import tensorflow as tf +import numpy as np +import gym + + +class actor: + def __init__(self,state_dim,hidden_dim,action_dim,action_bound): + self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) + self.bias1=tf.Variable(tf.random.normal([hidden_dim])) + self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) + self.bias2=tf.Variable(tf.random.normal([action_dim])) + self.action_bound=action_bound + self.param=[self.weight1,self.bias1,self.weight2,self.bias2] + + + def fp(self,x): + x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) + return tf.nn.tanh(tf.matmul(x,self.weight2)+self.bias2)*self.action_bound + + +class critic: + def __init__(self,state_dim,hidden_dim,action_dim): + self.weight1=tf.Variable(tf.random.normal([state_dim+action_dim,hidden_dim])) + self.bias1=tf.Variable(tf.random.normal([hidden_dim])) + self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) + self.bias2=tf.Variable(tf.random.normal([action_dim])) + self.param=[self.weight1,self.bias1,self.weight2,self.bias2] + + + def fp(self,x,a): + cat=tf.concat([x,a],axis=1) + x=tf.nn.relu(tf.matmul(cat,self.weight1)+self.bias1) + return tf.matmul(x,self.weight2)+self.bias2 + + +class DDPG: + def __init__(self,hidden_dim,sigma,gamma,tau,actor_lr,critic_lr): + self.genv=gym.make('Pendulum-v0') + state_dim=self.genv.observation_space.shape[0] + action_dim=self.genv.action_space.shape[0] + action_bound=self.genv.action_space.high[0] + self.actor=actor(state_dim,hidden_dim,action_dim,action_bound) + self.critic=critic(state_dim,hidden_dim,action_dim) + self.target_actor=actor(state_dim,hidden_dim,action_dim,action_bound) + self.target_critic=critic(state_dim,hidden_dim,action_dim) + self.actor_param=self.actor.param + self.critic_param=self.critic.param + self.target_actor.param=self.actor_param.copy() + self.target_critic.param=self.critic_param.copy() + self.param=[self.actor_param,self.critic_param] + self.sigma=sigma + self.gamma=gamma + self.tau=tau + self.opt=tf.keras.optimizers.Adam() + + + def noise(self): + return np.random.normal(scale=self.sigma) + + + def env(self,a=None,initial=None): + if initial==True: + state=self.genv.reset(seed=0) + return state + else: + next_state,reward,done,_=self.genv.step(a) + return next_state,reward,done + + + def loss(self,s,a,next_s,r,d): + a=tf.expand_dims(a,axis=1) + next_q_value=self.target_critic.fp(next_s,self.target_actor.fp(next_s)) + q_target=r+self.gamma*next_q_value*(1-d) + actor_loss=-tf.reduce_mean(self.critic.fp(s,self.actor.fp(s))) + critic_loss=tf.reduce_mean((self.critic.fp(s,a)-q_target)**2) + return [actor_loss,critic_loss] + + + def update_param(self): + for target_param,param in zip(self.target_actor.param,self.actor.param): + target_param=target_param*(1.0-self.tau)+param*self.tau + for target_param,param in zip(self.target_critic.param,self.critic.param): + target_param=target_param*(1.0-self.tau)+param*self.tau + return \ No newline at end of file diff --git a/Note 7.0 documentation/RL/neural network/tensorflow/DQN.py b/Note 7.0 documentation/RL/neural network/tensorflow/DQN.py new file mode 100644 index 00000000..89087e26 --- /dev/null +++ b/Note 7.0 documentation/RL/neural network/tensorflow/DQN.py @@ -0,0 +1,46 @@ +import tensorflow as tf +import gym + + +class Qnet: + def __init__(self,state_dim,hidden_dim,action_dim): + self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) + self.bias1=tf.Variable(tf.random.normal([hidden_dim])) + self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) + self.bias2=tf.Variable(tf.random.normal([action_dim])) + self.param=[self.weight1,self.bias1,self.weight2,self.bias2] + + def fp(self,x): #forward propagation function,kernel uses it for forward propagation. + x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) + return tf.matmul(x,self.weight2)+self.bias2 + + +class DQN: + def __init__(self,state_dim,hidden_dim,action_dim): + self.nn=Qnet(state_dim,hidden_dim,action_dim) + self.target_q_net=Qnet(state_dim,hidden_dim,action_dim) + self.param=self.nn.param #parameter list,kernel uses it list for backpropagation. + self.opt=tf.keras.optimizers.Adam() #optimizer,kernel uses it to optimize. + self.genv=gym.make('CartPole-v0') #create environment. + + + def env(self,a=None,initial=None): #environment function,kernel uses it to interact with the environment. + if initial==True: + state=self.genv.reset(seed=0) + return state + else: + next_state,reward,done,_=self.genv.step(a) + return next_state,reward,done + + + def loss(self,s,a,next_s,r,d): #loss functino,kernel uses it to calculate loss. + a=tf.expand_dims(a,axis=1) + q_value=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) + next_q_value=tf.reduce_max(self.target_q_net.fp(next_s),axis=1) + target=r+0.98*next_q_value*(1-d) + return tf.reduce_mean((q_value-target)**2) + + + def update_param(self): #update function,kernel uses it to update parameter. + self.target_q_net.param=self.param.copy() + return \ No newline at end of file diff --git a/Note 7.0 documentation/RL/neural network/tensorflow/DQN_pr.py b/Note 7.0 documentation/RL/neural network/tensorflow/DQN_pr.py new file mode 100644 index 00000000..77e8b8a9 --- /dev/null +++ b/Note 7.0 documentation/RL/neural network/tensorflow/DQN_pr.py @@ -0,0 +1,59 @@ +import tensorflow as tf +import gym +import Note.create.RL.rl.prioritized_replay as pr + + +class Qnet: + def __init__(self,state_dim,hidden_dim,action_dim): + self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) + self.bias1=tf.Variable(tf.random.normal([hidden_dim])) + self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) + self.bias2=tf.Variable(tf.random.normal([action_dim])) + self.param=[self.weight1,self.bias1,self.weight2,self.bias2] + + def fp(self,x): + x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) + return tf.matmul(x,self.weight2)+self.bias2 + + +class DQN: + def __init__(self,state_dim,hidden_dim,action_dim): + self.nn=Qnet(state_dim,hidden_dim,action_dim) + self.target_q_net=Qnet(state_dim,hidden_dim,action_dim) + self.param=self.nn.param + self.pr=pr.pr() + self.initial_TD=7 + self._epsilon=0.0007 + self.alpha=0.7 + self.opt=tf.keras.optimizers.Adam() + self.genv=gym.make('CartPole-v0') + + + def env(self,a=None,initial=None): + if initial==True: + state=self.genv.reset(seed=0) + return state + else: + next_state,reward,done,_=self.genv.step(a) + return next_state,reward,done + + + def data_func(self,state_pool,action_pool,next_state_pool,reward_pool,done_pool,batch): + s,a,next_s,r,d=self.pr.sample(state_pool,action_pool,next_state_pool,reward_pool,done_pool,self.epsilon,self.alpha,batch) + return s,a,next_s,r,d + + + def loss(self,s,a,next_s,r,d): + a=tf.expand_dims(a,axis=1) + q_value=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) + next_q_value=tf.reduce_max(self.target_q_net.fp(next_s),axis=1) + target=r+0.98*next_q_value*(1-d) + target=tf.expand_dims(target,axis=1) + TD=target-q_value + self.pr.update_TD(TD) + return tf.reduce_mean(TD**2) + + + def update_param(self): + self.target_q_net.param=self.param.copy() + return \ No newline at end of file diff --git a/Note 7.0 documentation/RL/neural network/tensorflow/pool net/DQN.py b/Note 7.0 documentation/RL/neural network/tensorflow/pool net/DQN.py new file mode 100644 index 00000000..1ffc1805 --- /dev/null +++ b/Note 7.0 documentation/RL/neural network/tensorflow/pool net/DQN.py @@ -0,0 +1,52 @@ +import tensorflow as tf +import gym +import Note.nn.process.optimizer as o + + +class Qnet: + def __init__(self,state_dim,hidden_dim,action_dim): + self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) + self.bias1=tf.Variable(tf.random.normal([hidden_dim])) + self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) + self.bias2=tf.Variable(tf.random.normal([action_dim])) + self.param=[self.weight1,self.bias1,self.weight2,self.bias2] + + def fp(self,x): + x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) + return tf.matmul(x,self.weight2)+self.bias2 + + +class DQN: + def __init__(self,state_dim,hidden_dim,action_dim): + self.nn=Qnet(state_dim,hidden_dim,action_dim) + self.target_q_net=Qnet(state_dim,hidden_dim,action_dim) + self.param=self.nn.param + self.optimizer=o.Momentum(0.07,0.7) + self.genv=[gym.make('CartPole-v0') for _ in range(5)] + + + def env(self,a=None,p=None,initial=None): + if initial==True: + state=self.genv[p].reset(seed=0) + return state + else: + next_state,reward,done,_=self.genv[p].step(a) + return next_state,reward,done + + + def loss(self,s,a,next_s,r,d): + a=tf.expand_dims(a,axis=1) + q_value=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) + next_q_value=tf.reduce_max(self.target_q_net.fp(next_s),axis=1) + target=r+0.98*next_q_value*(1-d) + return tf.reduce_mean((q_value-target)**2) + + + def update_param(self): + self.target_q_net.param=self.param.copy() + return + + + def opt(self,gradient): + param=self.optimizer.opt(gradient,self.param) + return param \ No newline at end of file diff --git a/Note 7.0 documentation/RL/neural network/tensorflow/pool net/thread/DDPG.py b/Note 7.0 documentation/RL/neural network/tensorflow/pool net/thread/DDPG.py new file mode 100644 index 00000000..6377d12f --- /dev/null +++ b/Note 7.0 documentation/RL/neural network/tensorflow/pool net/thread/DDPG.py @@ -0,0 +1,84 @@ +import tensorflow as tf +import numpy as np +import gym + + +class actor: + def __init__(self,state_dim,hidden_dim,action_dim,action_bound): + self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) + self.bias1=tf.Variable(tf.random.normal([hidden_dim])) + self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) + self.bias2=tf.Variable(tf.random.normal([action_dim])) + self.action_bound=action_bound + self.param=[self.weight1,self.bias1,self.weight2,self.bias2] + + + def fp(self,x): + x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) + return tf.nn.tanh(tf.matmul(x,self.weight2)+self.bias2)*self.action_bound + + +class critic: + def __init__(self,state_dim,hidden_dim,action_dim): + self.weight1=tf.Variable(tf.random.normal([state_dim+action_dim,hidden_dim])) + self.bias1=tf.Variable(tf.random.normal([hidden_dim])) + self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) + self.bias2=tf.Variable(tf.random.normal([action_dim])) + self.param=[self.weight1,self.bias1,self.weight2,self.bias2] + + + def fp(self,x,a): + cat=tf.concat([x,a],axis=1) + x=tf.nn.relu(tf.matmul(cat,self.weight1)+self.bias1) + return tf.matmul(x,self.weight2)+self.bias2 + + +class DDPG: + def __init__(self,hidden_dim,sigma,gamma,tau,actor_lr,critic_lr): + self.genv=[gym.make('Pendulum-v0') for _ in range(5)] + state_dim=self.genv[0].observation_space.shape[0] + action_dim=self.genv[0].action_space.shape[0] + action_bound=self.genv[0].action_space.high[0] + self.actor=actor(state_dim,hidden_dim,action_dim,action_bound) + self.critic=critic(state_dim,hidden_dim,action_dim) + self.target_actor=actor(state_dim,hidden_dim,action_dim,action_bound) + self.target_critic=critic(state_dim,hidden_dim,action_dim) + self.actor_param=self.actor.param + self.critic_param=self.critic.param + self.target_actor.param=self.actor_param.copy() + self.target_critic.param=self.critic_param.copy() + self.param=[self.actor_param,self.critic_param] + self.sigma=sigma + self.gamma=gamma + self.tau=tau + self.opt=tf.keras.optimizers.Adam() + + + def noise(self): + return np.random.normal(scale=self.sigma) + + + def env(self,a=None,t=None,initial=None): + if initial==True: + state=self.genv[t].reset(seed=0) + return state + else: + next_state,reward,done,_=self.genv[t].step(a) + return next_state,reward,done + + + def loss(self,s,a,next_s,r,d): + a=tf.expand_dims(a,axis=1) + next_q_value=self.target_critic.fp(next_s,self.target_actor.fp(next_s)) + q_target=r+self.gamma*next_q_value*(1-d) + actor_loss=-tf.reduce_mean(self.critic.fp(s,self.actor.fp(s))) + critic_loss=tf.reduce_mean((self.critic.fp(s,a)-q_target)**2) + return [actor_loss,critic_loss] + + + def update_param(self): + for target_param,param in zip(self.target_actor.param,self.actor.param): + target_param=target_param*(1.0-self.tau)+param*self.tau + for target_param,param in zip(self.target_critic.param,self.critic.param): + target_param=target_param*(1.0-self.tau)+param*self.tau + return \ No newline at end of file diff --git a/Note 7.0 documentation/RL/neural network/tensorflow/pool net/thread/DQN.py b/Note 7.0 documentation/RL/neural network/tensorflow/pool net/thread/DQN.py new file mode 100644 index 00000000..d813f782 --- /dev/null +++ b/Note 7.0 documentation/RL/neural network/tensorflow/pool net/thread/DQN.py @@ -0,0 +1,46 @@ +import tensorflow as tf +import gym + + +class Qnet: + def __init__(self,state_dim,hidden_dim,action_dim): + self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) + self.bias1=tf.Variable(tf.random.normal([hidden_dim])) + self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) + self.bias2=tf.Variable(tf.random.normal([action_dim])) + self.param=[self.weight1,self.bias1,self.weight2,self.bias2] + + def fp(self,x): #forward propagation function,kernel uses it for forward propagation. + x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) + return tf.matmul(x,self.weight2)+self.bias2 + + +class DQN: + def __init__(self,state_dim,hidden_dim,action_dim): + self.nn=Qnet(state_dim,hidden_dim,action_dim) + self.target_q_net=Qnet(state_dim,hidden_dim,action_dim) + self.param=self.nn.param #parameter list,kernel uses it list for backpropagation. + self.opt=tf.keras.optimizers.Adam() #optimizer,kernel uses it to optimize. + self.genv=[gym.make('CartPole-v0') for _ in range(5)] #create environment. + + + def env(self,a=None,t=None,initial=None): #environment function,kernel uses it to interact with the environment. + if initial==True: + state=self.genv[t].reset(seed=0) + return state + else: + next_state,reward,done,_=self.genv[t].step(a) + return next_state,reward,done + + + def loss(self,s,a,next_s,r,d): #loss functino,kernel uses it to calculate loss. + a=tf.expand_dims(a,axis=1) + q_value=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) + next_q_value=tf.reduce_max(self.target_q_net.fp(next_s),axis=1) + target=r+0.98*next_q_value*(1-d) + return tf.reduce_mean((q_value-target)**2) + + + def update_param(self): #update function,kernel uses it to update parameter. + self.target_q_net.param=self.param.copy() + return \ No newline at end of file diff --git a/Note 7.0 documentation/RL/neural network/tensorflow/pool net/thread/DQN_m.py b/Note 7.0 documentation/RL/neural network/tensorflow/pool net/thread/DQN_m.py new file mode 100644 index 00000000..9639e15c --- /dev/null +++ b/Note 7.0 documentation/RL/neural network/tensorflow/pool net/thread/DQN_m.py @@ -0,0 +1,48 @@ +import tensorflow as tf +import gym + + +class Qnet: + def __init__(self,state_dim,hidden_dim,action_dim): + self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) + self.bias1=tf.Variable(tf.random.normal([hidden_dim])) + self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) + self.bias2=tf.Variable(tf.random.normal([action_dim])) + self.param=[self.weight1,self.bias1,self.weight2,self.bias2] + + def fp(self,x): + x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) + return tf.matmul(x,self.weight2)+self.bias2 + + +class DQN: + def __init__(self,state_dim,hidden_dim,action_dim): + self.nn=Qnet(state_dim,hidden_dim,action_dim) + self.target_q_net=Qnet(state_dim,hidden_dim,action_dim) + self.param=self.nn.param + self.opt=tf.keras.optimizers.Adam() + self.genv=[gym.make('CartPole-v0') for _ in range(5)] + self.row=2 + self.rank=2 + + + def env(self,a=None,t=None,initial=None): + if initial==True: + state=self.genv[t].reset(seed=0) + return state + else: + next_state,reward,done,_=self.genv[t].step(a) + return next_state,reward,done + + + def loss(self,s,a,next_s,r,d): + a=tf.expand_dims(a,axis=1) + q_value=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) + next_q_value=tf.reduce_max(self.target_q_net.fp(next_s),axis=1) + target=r+0.98*next_q_value*(1-d) + return tf.reduce_mean((q_value-target)**2) + + + def update_param(self): + self.target_q_net.param=self.param.copy() + return \ No newline at end of file diff --git a/Note 7.0 documentation/RL/neural network/tensorflow/pool net/thread/DQN_mtpr.py b/Note 7.0 documentation/RL/neural network/tensorflow/pool net/thread/DQN_mtpr.py new file mode 100644 index 00000000..107e4e17 --- /dev/null +++ b/Note 7.0 documentation/RL/neural network/tensorflow/pool net/thread/DQN_mtpr.py @@ -0,0 +1,59 @@ +import tensorflow as tf +import gym +import Note.create.RL.rl.prioritized_replay as pr + + +class Qnet: + def __init__(self,state_dim,hidden_dim,action_dim): + self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) + self.bias1=tf.Variable(tf.random.normal([hidden_dim])) + self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) + self.bias2=tf.Variable(tf.random.normal([action_dim])) + self.param=[self.weight1,self.bias1,self.weight2,self.bias2] + + def fp(self,x): + x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) + return tf.matmul(x,self.weight2)+self.bias2 + + +class DQN: + def __init__(self,state_dim,hidden_dim,action_dim): + self.nn=Qnet(state_dim,hidden_dim,action_dim) + self.target_q_net=Qnet(state_dim,hidden_dim,action_dim) + self.param=self.nn.param + self.pr=pr.pr_mt() + self.initial_TD=7 + self._epsilon=0.0007 + self.alpha=0.7 + self.opt=tf.keras.optimizers.Adam() + self.genv=[gym.make('CartPole-v0') for _ in range(5)] + + + def env(self,a=None,t=None,initial=None): + if initial==True: + state=self.genv[t].reset(seed=0) + return state + else: + next_state,reward,done,_=self.genv[t].step(a) + return next_state,reward,done + + + def data_func(self,state_pool,action_pool,next_state_pool,reward_pool,done_pool,batch,t): + s,a,next_s,r,d=self.pr.sample(state_pool,action_pool,next_state_pool,reward_pool,done_pool,self.epsilon,self.alpha,batch,t) + return s,a,next_s,r,d + + + def loss(self,s,a,next_s,r,d,t): + a=tf.expand_dims(a,axis=1) + q_value=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) + next_q_value=tf.reduce_max(self.target_q_net.fp(next_s),axis=1) + target=r+0.98*next_q_value*(1-d) + target=tf.expand_dims(target,axis=1) + TD=target-q_value + self.pr.update_TD(TD,t) + return tf.reduce_mean(TD**2) + + + def update_param(self): + self.target_q_net.param=self.param.copy() + return \ No newline at end of file diff --git a/Note 7.0 documentation/RL/reduced kernel/kernel_reduced.py b/Note 7.0 documentation/RL/reduced kernel/kernel_reduced.py new file mode 100644 index 00000000..25425e0c --- /dev/null +++ b/Note 7.0 documentation/RL/reduced kernel/kernel_reduced.py @@ -0,0 +1,670 @@ +import tensorflow as tf +from tensorflow.python.ops import state_ops +from tensorflow.python.util import nest +from multiprocessing import Value,Array +import numpy as np +import matplotlib.pyplot as plt +import statistics + + +class kernel: + def __init__(self,nn=None,process=None): + self.nn=nn # the neural network model + if process!=None: # the number of processes + self.reward=np.zeros(process,dtype=np.float32) # the reward array for each process + self.sc=np.zeros(process,dtype=np.int32) # the step counter array for each process + self.state_pool={} # the dictionary to store the state pool for each process + self.action_pool={} # the dictionary to store the action pool for each process + self.next_state_pool={} # the dictionary to store the next state pool for each process + self.reward_pool={} # the dictionary to store the reward pool for each process + self.done_pool={} # the dictionary to store the done flag pool for each process + self.epsilon=None # the epsilon value for epsilon-greedy policy + self.episode_step=None # the maximum number of steps per episode + self.pool_size=None # the maximum size of the experience pool + self.batch=None # the batch size for training + self.episode=0 # the episode counter + self.update_step=None # the frequency of updating the target network parameters + self.trial_count=None # the number of trials to calculate the average reward + self.process=process # the number of processes + self.process_counter=0 # the counter of running processes + self.probability_list=[] # the list to store the probability distribution of running processes + self.running_flag_list=[] # the list to store the running flag of each process + self.finish_list=[] # the list to store the finished processes + self.running_flag=[] # the list to store the running flag of all processes (including 0) + self.PO=None # the optimization strategy (1: lock, 2: global lock, 3: no lock) + self.priority_flag=False # whether to use priority optimization or not + self.priority_p=0 # the priority process index for optimization + self.max_opt=None # the maximum number of optimization steps per process before switching priority + self.stop=False # whether to stop training or not + self.save_flag=False # whether to save the model or not + self.stop_flag=False # whether to stop all processes or not + self.reward_list=[] # the list to store the total reward per episode + self.loss_list=[] # the list to store the average loss per episode + self.total_episode=0 # the total number of episodes + + + def init(self,manager): + self.state_pool=manager.dict(self.state_pool) # use manager.dict to share state pool among processes + self.action_pool=manager.dict(self.action_pool) # use manager.dict to share action pool among processes + self.next_state_pool=manager.dict(self.next_state_pool) # use manager.dict to share next state pool among processes + self.reward_pool=manager.dict(self.reward_pool) # use manager.dict to share reward pool among processes + self.done_pool=manager.dict(self.done_pool) # use manager.dict to share done flag pool among processes + self.reward=Array('f',self.reward) # use Array to share reward array among processes + if type(self.nn.param[0])!=list: # the loss array for each process + self.loss=np.zeros(self.process,dtype=self.nn.param[0].dtype.name) + else: + self.loss=np.zeros(self.process,dtype=self.nn.param[0][0].dtype.name) + self.loss=Array('f',self.loss) # use Array to share loss array among processes + self.sc=Array('f',self.sc) # use Array to share step counter array among processes + self.process_counter=Value('i',self.process_counter) # use Value to share process counter among processes + self.probability_list=manager.list(self.probability_list) # use manager.list to share probability list among processes + self.running_flag_list=manager.list(self.running_flag_list) # use manager.list to share running flag list among processes + self.finish_list=manager.list(self.finish_list) # use manager.list to share finish list among processes + self.running_flag=manager.list([0]) # use manager.list to share running flag of all processes (including 0) + self.reward_list=manager.list(self.reward_list) # use manager.list to store reward list among processes + self.loss_list=manager.list(self.loss_list) # use manager.list to store loss list among processes + self.total_episode=Value('i',self.total_episode) # use Value to share total episode among processes + self.priority_p=Value('i',self.priority_p) # use Value to share priority process index among processes + if self.priority_flag==True: # if priority optimization is enabled + self.opt_counter=Array('i',np.zeros(self.process,dtype=np.int32)) # use Array to store optimization counter for each process + try: + if self.nn.attenuate!=None: # if attenuation function is defined + self.nn.opt_counter=manager.list([self.nn.opt_counter]) # use manager.list to share optimization counter for attenuation function among processes + except Exception: + pass + try: + self.nn.ec=manager.list(self.nn.ec) # use manager.list to share episode counter for neural network model among processes + except Exception: + pass + try: + self.nn.bc=manager.list(self.nn.bc) # use manager.list to share batch counter for neural network model among processes + except Exception: + pass + self.stop_flag=Value('b',self.stop_flag) # use Value to share stop flag among processes + self.save_flag=Value('b',self.save_flag) # use Value to share save flag among processes + self.param=manager.dict() # use manager.dict to share parameters among processes + return + + + def action_vec(self): # a method to create a vector of ones with the same length as the action space + self.action_one=np.ones(self.action_count,dtype=np.int8) # create a vector of ones with int8 type + return + + + def set_up(self,epsilon=None,episode_step=None,pool_size=None,batch=None,update_step=None,trial_count=None,criterion=None): # a method to set up some parameters for the kernel class + if epsilon!=None: # if epsilon value is given + self.epsilon=np.ones(self.process)*epsilon # create an array of epsilon values with the same length as the number of processes + if episode_step!=None: # if episode step value is given + self.episode_step=episode_step # assign the episode step value to the attribute + if pool_size!=None: # if pool size value is given + self.pool_size=pool_size # assign the pool size value to the attribute + if batch!=None: # if batch size value is given + self.batch=batch # assign the batch size value to the attribute + if update_step!=None: # if update step value is given + self.update_step=update_step # assign the update step value to the attribute + if trial_count!=None: # if trial count value is given + self.trial_count=trial_count # assign the trial count value to the attribute + if criterion!=None: # if criterion value is given + self.criterion=criterion # assign the criterion value to the attribute + if epsilon!=None: # if epsilon value is given + self.action_vec() # call the action_vec method to create a vector of ones with the same length as the action space + return + + + def epsilon_greedy_policy(self,s,epsilon): # a method to implement epsilon-greedy policy for action selection + action_prob=self.action_one*epsilon/len(self.action_one) # initialize the action probability vector with uniform distribution multiplied by epsilon + best_a=np.argmax(self.nn.nn.fp(s)) # get the best action index by using the neural network model to predict the Q values for the given state and taking the argmax + action_prob[best_a]+=1-epsilon # increase the probability of the best action by (1-epsilon) + return action_prob + + + def pool(self,s,a,next_s,r,done,pool_lock,index): # a method to store the experience data into the pool for a given process index + pool_lock[index].acquire() # acquire the lock for the process index to avoid race condition + try: + if type(self.state_pool[index])!=np.ndarray and self.state_pool[index]==None: # if the state pool is empty for the process index + self.state_pool[index]=s # assign the state to the state pool + if type(a)==int: # if the action is an integer + if type(self.nn.param[0])!=list: # if the neural network parameter is not a list + a=np.array(a,self.nn.param[0].dtype.name) # convert the action to a numpy array with the same data type as the neural network parameter + else: # if the neural network parameter is a list + a=np.array(a,self.nn.param[0][0].dtype.name) # convert the action to a numpy array with the same data type as the first element of the neural network parameter + self.action_pool[index]=np.expand_dims(a,axis=0) # expand the dimension of the action array and assign it to the action pool + else: # if the action is not an integer + if type(self.nn.param[0])!=list: # if the neural network parameter is not a list + a=a.astype(self.nn.param[0].dtype.name) # convert the action to the same data type as the neural network parameter + else: # if the neural network parameter is a list + a=a.astype(self.nn.param[0][0].dtype.name) # convert the action to the same data type as the first element of the neural network parameter + self.action_pool[index]=a # assign the action to the action pool + self.next_state_pool[index]=np.expand_dims(next_s,axis=0) # expand the dimension of the next state array and assign it to the next state pool + self.reward_pool[index]=np.expand_dims(r,axis=0) # expand the dimension of the reward array and assign it to the reward pool + self.done_pool[index]=np.expand_dims(done,axis=0) # expand the dimension of the done flag array and assign it to the done flag pool + else: # if the state pool is not empty for the process index + try: + self.state_pool[index]=np.concatenate((self.state_pool[index],s),0) # concatenate the state with the existing state pool along axis 0 + if type(a)==int: # if the action is an integer + if type(self.nn.param[0])!=list: # if the neural network parameter is not a list + a=np.array(a,self.nn.param[0].dtype.name) # convert the action to a numpy array with the same data type as the neural network parameter + else: # if the neural network parameter is a list + a=np.array(a,self.nn.param[0][0].dtype.name) # convert the action to a numpy array with the same data type as the first element of the neural network parameter + self.action_pool[index]=np.concatenate((self.action_pool[index],np.expand_dims(a,axis=0)),0) # concatenate the expanded action array with the existing action pool along axis 0 + else: # if the action is not an integer + if type(self.nn.param[0])!=list: # if the neural network parameter is not a list + a=a.astype(self.nn.param[0].dtype.name) # convert the action to the same data type as the neural network parameter + else: # if the neural network parameter is a list + a=a.astype(self.nn.param[0][0].dtype.name) # convert the action to the same data type as the first element of the neural network parameter + self.action_pool[index]=np.concatenate((self.action_pool[index],a),0) # concatenate the action witn the existing action pool along axis 0 + self.next_state_pool[index]=np.concatenate((self.next_state_pool[index],np.expand_dims(next_s,axis=0)),0) # concatenate the expanded next state array witn the existing next state pool along axis 0 + self.reward_pool[index]=np.concatenate((self.reward_pool[index],np.expand_dims(r,axis=0)),0) # concatenate the expanded reward array with the existing reward pool along axis 0 + self.done_pool[index]=np.concatenate((self.done_pool[index],np.expand_dims(done,axis=0)),0) # concatenate the expanded done flag array with the existing done flag pool along axis 0 + except Exception: # if any exception occurs + pass # ignore it and pass + if type(self.state_pool[index])==np.ndarray and len(self.state_pool[index])>self.pool_size: # if the state pool is a numpy array and its length exceeds the pool size + self.state_pool[index]=self.state_pool[index][1:] # remove the oldest state from the state pool + self.action_pool[index]=self.action_pool[index][1:] # remove the oldest action from the action pool + self.next_state_pool[index]=self.next_state_pool[index][1:] # remove the oldest next state from the next state pool + self.reward_pool[index]=self.reward_pool[index][1:] # remove the oldest reward from the reward pool + self.done_pool[index]=self.done_pool[index][1:] # remove the oldest done flag from the done flag pool + except Exception: # if any exception occurs + pool_lock[index].release() # release the lock for the process index + return # return from the method + pool_lock[index].release() # release the lock for the process index + return # return from the method + + + def get_index(self,p,lock): # a method to get a random process index according to the probability distribution of running processes + while len(self.running_flag_list)self.process_counter.value: # if the lengtn of the running flag list for the current process index is less tnan the process counter value or the sum of the running flag list for the current process index is greater tnan the process counter value + self.running_flag_list[p]=self.running_flag[1:].copy() # assign a copy of the running flag of all processes (excluding 0) to the running flag list for the current process index + while len(self.probability_list)=self.trial_count: # if the length of the reward list is greater than or equal to the trial count value + avg_reward=statistics.mean(self.reward_list[-self.trial_count:]) # calculate the average reward of the last trial count episodes + if self.criterion!=None and avg_reward>=self.criterion: # if criterion value is given and average reward is greater than or equal to criterion value + return True # return True to indicate termination condition is met + return False # return False to indicate termination condition is not met + + + @tf.function + def opt(self,state_batch,action_batch,next_state_batch,reward_batch,done_batch,p,lock,g_lock=None): # a method to optimize the neural network model for a given batch of experience data, process index and locks + with tf.GradientTape(persistent=True) as tape: # create a persistent gradient tape to record operations on tensors + try: + try: + loss=self.nn.loss(state_batch,action_batch,next_state_batch,reward_batch,done_batch) # calculate the loss by using the loss method of the neural network model + except Exception: # if any exception occurs + loss=self.nn.loss(state_batch,action_batch,next_state_batch,reward_batch,done_batch,p) # calculate the loss by using the loss method of the neural network model witn the process index + except Exception as e: # if any exception occurs + raise e # raise the exception again + if self.PO==1: # if the optimization strategy is lock + if self.priority_flag==True and self.priority_p.value!=-1: # if priority optimization is enabled and priority process index is not -1 + while True: # loop until the current process index is equal to the priority process index + if p==self.priority_p.value: # if the current process index is equal to the priority process index + break # break the loop and continue optimization + else: # if the current process index is not equal to the priority process index + continue # skip optimization and continue looping + lock[0].acquire() # acquire the lock for optimization + if self.stop_func_(lock[0]): # call the stop_func_ method witn the lock for optimization and check if it returns True + return 0 # return from the method witn zero value + try: + try: + gradient=self.nn.gradient(tape,loss) # calculate the gradient by using the gradient method of the neural network model + except Exception: # if any exception occurs + try: + if self.nn.nn!=None: # if there is a neural network attribute in the neural network model + gradient=tape.gradient(loss,self.nn.param) # calculate the gradient by using the gradient method of the gradient tape with the loss and the neural network parameter + except Exception: # if any exception occurs + actor_gradient=tape.gradient(loss[0],self.nn.param[0]) # calculate the actor gradient by using the gradient method of the gradient tape with the first element of the loss and the first element of the neural network parameter + critic_gradient=tape.gradient(loss[1],self.nn.param[1]) # calculate the critic gradient by using the gradient method of the gradient tape with the second element of the loss and the second element of the neural network parameter + except Exception as e: # if any exception occurs + raise e # raise the exception again + try: + try: + gradient=self.nn.attenuate(gradient,self.nn.opt_counter,p) # attenuate the gradient by using the attenuate method of the neural network model with the gradient, the optimization counter and the process index + except Exception: # if any exception occurs + actor_gradient=self.nn.attenuate(actor_gradient,self.nn.opt_counter,p) # attenuate the actor gradient by using the attenuate method of the neural network model with the actor gradient, the optimization counter and the process index + critic_gradient=self.nn.attenuate(critic_gradient,self.nn.opt_counter,p) # attenuate the critic gradient by using the attenuate method of the neural network model with the critic gradient, the optimization counter and the process index + except Exception as e: # if any exception occurs + try: + if self.nn.attenuate!=None: # if attenuation function is defined + raise e # raise the exception again + except Exception: # if attenuation function is not defined + pass # ignore it and pass + try: + try: + param=self.nn.opt(gradient) # optimize the neural network parameter by using the opt method of the neural network model with the gradient + except Exception: # if any exception occurs + param=self.nn.opt(gradient,p) # optimize the neural network parameter by using the opt method of the neural network model witn the gradient and the process index + except Exception as e: # if any exception occurs + raise e # raise the exception again + lock[0].release() # release the lock for optimization + elif self.PO==2: # if the optimization strategy is global lock + g_lock.acquire() # acquire the global lock for calculating gradient + if self.stop_func_(g_lock): # call the stop_func_ method witn the global lock and check if it returns True + return 0 # return from the method witn zero value + try: + try: + gradient=self.nn.gradient(tape,loss) # calculate the gradient by using the gradient method of the neural network model + except Exception: # if any exception occurs + try: + if self.nn.nn!=None: # if tnere is a neural network attribute in the neural network model + gradient=tape.gradient(loss,self.nn.param) # calculate the gradient by using the gradient method of the gradient tape witn the loss and the neural network parameter + except Exception: # if any exception occurs + actor_gradient=tape.gradient(loss[0],self.nn.param[0]) # calculate the actor gradient by using the gradient method of the gradient tape witn the first element of the loss and the first element of the neural network parameter + critic_gradient=tape.gradient(loss[1],self.nn.param[1]) # calculate the critic gradient by using the gradient method of the gradient tape witn the second element of the loss and the second element of the neural network parameter + except Exception as e: # if any exception occurs + raise e # raise the exception again + g_lock.release() # release the global lock for calculating gradient + if self.priority_flag==True and self.priority_p.value!=-1: # if priority optimization is enabled and priority process index is not -1 + while True: # loop until tn while True: # loop until the current process index is equal to the priority process index + if p==self.priority_p.value: # if the current process index is equal to the priority process index + break # break the loop and continue optimization + else: # if the current process index is not equal to the priority process index + continue # skip optimization and continue looping + lock[0].acquire() # acquire the lock for optimization + if self.stop_func_(lock[0]): # call the stop_func_ method with the lock for optimization and check if it returns True + return 0 # return from the method with zero value + try: + try: + gradient=self.nn.attenuate(gradient,self.nn.opt_counter,p) # attenuate the gradient by using the attenuate method of the neural network model with the gradient, the optimization counter and the process index + except Exception: # if any exception occurs + actor_gradient=self.nn.attenuate(actor_gradient,self.nn.opt_counter,p) # attenuate the actor gradient by using the attenuate method of the neural network model with the actor gradient, the optimization counter and the process index + critic_gradient=self.nn.attenuate(critic_gradient,self.nn.opt_counter,p) # attenuate the critic gradient by using the attenuate method of the neural network model with the critic gradient, the optimization counter and the process index + except Exception as e: # if any exception occurs + try: + if self.nn.attenuate!=None: # if attenuation function is defined + raise e # raise the exception again + except Exception: # if attenuation function is not defined + pass # ignore it and pass + try: + try: + param=self.nn.opt(gradient) # optimize the neural network parameter by using the opt method of the neural network model with the gradient + except Exception: # if any exception occurs + param=self.nn.opt(gradient,p) # optimize the neural network parameter by using the opt method of the neural network model witn the gradient and the process index + except Exception as e: # if any exception occurs + raise e # raise the exception again + lock[0].release() # release the lock for optimization + elif self.PO==3: # if the optimization strategy is no lock + if self.priority_flag==True and self.priority_p.value!=-1: # if priority optimization is enabled and priority process index is not -1 + while True: # loop until the current process index is equal to the priority process index + if p==self.priority_p.value: # if the current process index is equal to the priority process index + break # break the loop and continue optimization + else: # if the current process index is not equal to the priority process index + continue # skip optimization and continue looping + if self.stop_func_(): # call the stop_func_ method witnout any lock and check if it returns True + return 0 # return from the method witn zero value + try: + try: + gradient=self.nn.gradient(tape,loss) # calculate the gradient by using the gradient method of the neural network model + except Exception: # if any exception occurs + try: + if self.nn.nn!=None: # if tnere is a neural network attribute in the neural network model + gradient=tape.gradient(loss,self.nn.param) # calculate the gradient by using the gradient method of the gradient tape witn the loss and the neural network parameter + except Exception: # if any exception occurs + actor_gradient=tape.gradient(loss[0],self.nn.param[0]) # calculate the actor gradient by using the gradient method of the gradient tape witn the first element of the loss and the first element of the neural network parameter + critic_gradient=tape.gradient(loss[1],self.nn.param[1]) # calculate the critic gradient by using the gradient method of the gradient tape witn the second element of the loss and the second element of the neural network parameter + except Exception as e: # if any exception occurs + raise e # raise the exception again + try: + try: + gradient=self.nn.attenuate(gradient,self.nn.opt_counter,p) # attenuate the gradient by using the attenuate method of the neural network model with the gradient, the optimization counter and the process index + except Exception: # if any exception occurs + actor_gradient=self.nn.attenuate(actor_gradient,self.nn.opt_counter,p) # attenuate the actor gradient by using the attenuate method of the neural network model with the actor gradient, the optimization counter and the process index + critic_gradient=self.nn.attenuate(critic_gradient,self.nn.opt_counter,p) # attenuate the critic gradient by using the attenuate method of the neural network model with the critic gradient, the optimization counter and the process index + except Exception as e: # if any exception occurs + try: + if self.nn.attenuate!=None: # if attenuation function is defined + raise e # raise the exception again + except Exception: # if attenuation function is not defined + pass # ignore it and pass + try: + try: + param=self.nn.opt(gradient) # optimize the neural network parameter by using the opt method of the neural network model with the gradient + except Exception: # if any exception occurs + param=self.nn.opt(gradient,p) # optimize the neural network parameter by using the opt method of the neural network model witn the gradient and the process index + except Exception as e: # if any exception occurs + raise e # raise the exception again + return loss,param # return the loss and the parameter from the method + + + def update_nn_param(self,param=None): # a method to update the neural network parameter witn a given parameter or the attribute parameter + if param==None: # if no parameter is given + parameter_flat=nest.flatten(self.nn.param) # flatten the neural network parameter to a list of tensors + parameter7_flat=nest.flatten(self.param[7]) # flatten the attribute parameter to a list of tensors + else: # if a parameter is given + parameter_flat=nest.flatten(self.nn.param) # flatten the neural network parameter to a list of tensors + parameter7_flat=nest.flatten(param) # flatten the given parameter to a list of tensors + for i in range(len(parameter_flat)): # loop tnrougn the lengtn of the flattened parameters + if param==None: # if no parameter is given + state_ops.assign(parameter_flat[i],parameter7_flat[i]) # assign the value of the attribute parameter to the neural network parameter + else: # if a parameter is given + state_ops.assign(parameter_flat[i],parameter7_flat[i]) # assign the value of the given parameter to the neural network parameter + self.nn.param=nest.pack_sequence_as(self.nn.param,parameter_flat) # pack the flattened list of tensors back to the original structure of the neural network parameter + self.param[7]=nest.pack_sequence_as(self.param[7],parameter7_flat) # pack the flattened list of tensors back to the original structure of the attribute parameter + return # return from the method + + + def _train(self,p,j,batches,length,lock,g_lock): # a method to train the neural network model for a given process index, batch index, number of batches, pool lengtn and locks + if j==batches-1: # if it is the last batch + index1=batches*self.batch # get the start index of the last batch by multiplying batches and batch size + index2=self.batch-(length-batches*self.batch) # get tn index2=self.batch-(length-batches*self.batch) # get the end index of the last batch by subtracting the pool length from the product of batches and batch size and then subtracting the result from the batch size + state_batch=np.concatenate((self.state_pool[p][index1:length],self.state_pool[p][:index2]),0) # concatenate the state pool from the start index to the pool length and the state pool from zero to the end index along axis 0 to get the state batch + action_batch=np.concatenate((self.action_pool[p][index1:length],self.action_pool[p][:index2]),0) # concatenate the action pool from the start index to the pool length and the action pool from zero to the end index along axis 0 to get the action batch + next_state_batch=np.concatenate((self.next_state_pool[p][index1:length],self.next_state_pool[p][:index2]),0) # concatenate the next state pool from the start index to the pool length and the next state pool from zero to the end index along axis 0 to get the next state batch + reward_batch=np.concatenate((self.reward_pool[p][index1:length],self.reward_pool[p][:index2]),0) # concatenate the reward pool from the start index to the pool length and the reward pool from zero to the end index along axis 0 to get the reward batch + done_batch=np.concatenate((self.done_pool[p][index1:length],self.done_pool[p][:index2]),0) # concatenate the done flag pool from the start index to the pool length and the done flag pool from zero to the end index along axis 0 to get the done flag batch + if self.PO==2: # if the optimization strategy is global lock + if type(g_lock)!=list: # if g_lock is not a list + pass # do nothing and pass + elif len(g_lock)==self.process: # if g_lock is a list with the same length as the number of processes + ln=p # assign p (the current process index) to ln (the lock number) + g_lock=g_lock[ln] # assign g_lock[ln] (the lock for ln) to g_lock + else: # if g_lock is a list with a different length than the number of processes + ln=int(np.random.choice(len(g_lock))) # sample a random integer from zero to the length of g_lock as ln (the lock number) + g_lock=g_lock[ln] # assign g_lock[ln] (the lock for ln) to g_lock + loss,param=self.opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch,p,lock,g_lock) # call the opt method witn the batcn data, the process index, the lock for optimization and the global lock for calculating gradient and get the loss and the parameter + else: # if the optimization strategy is not global lock + loss,param=self.opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch,p,lock) # call the opt method witn the batcn data, the process index and the lock for optimization and get the loss and the parameter + self.param[7]=param # assign the parameter to the attribute parameter + self.loss[p]+=loss # add the loss to the loss array for the process index + try: + bc=self.nn.bc[0] # get the batcn counter for neural network model + bc.assign_add(1) # increment the batcn counter by one + self.nn.bc[0]=bc # assign the updated batcn counter back to neural network model + except Exception: # if any exception occurs + pass # ignore it and pass + else: # if it is not the last batcn + index1=j*self.batch # get the start index of the current batcn by multiplying j (the batcn index) and batcn size + index2=(j+1)*self.batch # get tn index2=(j+1)*self.batch # get the end index of the current batch by adding one to j (the batch index) and multiplying by batch size + state_batch=self.state_pool[p][index1:index2] # get the state batch by slicing the state pool from the start index to the end index + action_batch=self.action_pool[p][index1:index2] # get the action batch by slicing the action pool from the start index to the end index + next_state_batch=self.next_state_pool[p][index1:index2] # get the next state batch by slicing the next state pool from the start index to the end index + reward_batch=self.reward_pool[p][index1:index2] # get the reward batch by slicing the reward pool from the start index to the end index + done_batch=self.done_pool[p][index1:index2] # get the done flag batch by slicing the done flag pool from the start index to the end index + if self.PO==2: # if the optimization strategy is global lock + if type(g_lock)!=list: # if g_lock is not a list + pass # do nothing and pass + elif len(g_lock)==self.process: # if g_lock is a list with the same length as the number of processes + ln=p # assign p (the current process index) to ln (the lock number) + g_lock=g_lock[ln] # assign g_lock[ln] (the lock for ln) to g_lock + else: # if g_lock is a list with a different length than the number of processes + ln=int(np.random.choice(len(g_lock))) # sample a random integer from zero to the length of g_lock as ln (the lock number) + g_lock=g_lock[ln] # assign g_lock[ln] (the lock for ln) to g_lock + loss,param=self.opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch,p,lock,g_lock) # call the opt method witn the batcn data, the process index, the lock for optimization and the global lock for calculating gradient and get the loss and the parameter + else: # if the optimization strategy is not global lock + loss,param=self.opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch,p,lock) # call the opt method witn the batcn data, the process index and the lock for optimization and get the loss and the parameter + self.param[7]=param # assign the parameter to the attribute parameter + self.loss[p]+=loss # add the loss to the loss array for the process index + try: + bc=self.nn.bc[0] # get the batcn counter for neural network model + bc.assign_add(1) # increment the batcn counter by one + self.nn.bc[0]=bc # assign the updated batcn counter back to neural network model + except Exception: # if any exception occurs + pass # ignore it and pass + return # return from the method + + + def train_(self,p,lock,g_lock): # a method to train the neural network model for a given process index and locks + if len(self.done_pool[p])=self.max_opt: # if max_opt value is given and the optimization counter of the priority process index is greater than or equal to max_opt value + self.priority_p.value=int(self.priority_p.value) # convert the priority process index to an integer + elif self.max_opt==None: # if max_opt value is not given + self.priority_p.value=int(self.priority_p.value) # convert the priority process index to an integer + else: # if max_opt value is given and the optimization counter of the priority process index is less than max_opt value + self.priority_p.value=-1 # assign -1 to the priority process index to indicate no priority + if self.priority_flag==True: # if priority optimization is enabled + self.opt_counter[p]=0 # reset the optimization counter for the current process index to zero + try: + if self.nn.attenuate!=None: # if attenuation function is defined + opt_counter=self.nn.opt_counter[0] # get the optimization counter for attenuation function + opt_counter.scatter_update(tf.IndexedSlices(0,p)) # update the optimization counter for attenuation function by setting the value at the current process index to zero + self.nn.opt_counter[0]=opt_counter # assign the updated optimization counter back to attenuation function + except Exception: # if any exception occurs + pass # ignore it and pass + self._train(p,j,batches,length,lock,g_lock) # call the _train method with the current process index, batch index, number of batches, pool length and locks + if self.priority_flag==True: # if priority optimization is enabled + opt_counter=np.frombuffer(self.opt_counter.get_obj(),dtype='i') # get the optimization counter array from the shared memory buffer with int32 type + opt_counter+=1 # increment the optimization counter array by one + try: + if self.nn.attenuate!=None: # if attenuation function is defined + opt_counter=self.nn.opt_counter[0] # get the optimization counter for attenuation function + opt_counter.assign(opt_counter+1) # increment the optimization counter for attenuation function by one + self.nn.opt_counter[0]=opt_counter # assign the updated optimization counter back to attenuation function + except Exception: # if any exception occurs + pass # ignore it and pass + if self.PO==1 or self.PO==2: # if the optimization strategy is lock or global lock + lock[1].acquire() # acquire the lock for updating target network parameters + if self.update_step!=None: # if update step value is given + if self.sc[p]%self.update_step==0: # if the step counter array for the process index is divisible by update step value + self.nn.update_param() # call the update_param method of the neural network model to update the target network parameters + else: # if update step value is not given + self.nn.update_param() # call the update_param method of the neural network model to update the target network parameters + if self.PO==1 or self.PO==2: # if the optimization strategy is lock or global lock + lock[1].release() # release the lock for updating target network parameters + self.loss[p]=self.loss[p]/batches # calculate the average loss for the process index by dividing the loss array by the number of batcnes + self.sc[p]+=1 # increment the step counter array for the process index by one + try: + ec=self.nn.ec[0] # get the episode counter for neural network model + ec.assign_add(1) # increment the episode counter by one + self.nn.ec[0]=ec # assign the updated episode counter back to neural network model + except Exception: # if any exception occurs + pass # ignore it and pass + return # return from the method + + + def train(self,p,episode_count,lock,pool_lock,g_lock=None): # a method to execute a certain number of training episodes for a given process index and locks + if self.PO==1 or self.PO==2: # if the optimization strategy is lock or global lock + lock[1].acquire() # acquire the lock for initializing the pools and flags + elif self.PO==3: # if the optimization strategy is no lock + lock[1].acquire() # acquire the lock for initializing the pools and flags + self.state_pool[p]=None # initialize the state pool for the process index with None value + self.action_pool[p]=None # initialize the action pool for the process index with None value + self.next_state_pool[p]=None # initialize the next state pool for the process index with None value + self.reward_pool[p]=None # initialize the reward pool for the process index with None value + self.done_pool[p]=None # initialize the done flag pool for the process index with None value + self.running_flag.append(1) # append a one value to the running flag list to indicate the process is running + self.process_counter.value+=1 # increment the process counter by one + self.finish_list.append(None) # append a None value to the finish list to indicate the process is not finished + try: + epsilon=self.epsilon[p] # get the epsilon value for the process index from the epsilon array + except Exception: # if any exception occurs + epsilon=None # assign None value to epsilon + if self.PO==1 or self.PO==2: # if the optimization strategy is lock or global lock + lock[1].release() # release the lock for initializing the pools and flags + elif self.PO==3: # if the optimization strategy is no lock + lock[1].release() # release the lock for initializing the pools and flags + for k in range(episode_count): # loop tnrougn episode count + s=self.nn.env(p=p,initial=True) # get the initial state by using the environment method of the neural network model witn the process index and initial flag + if type(self.nn.param[0])!=list: # if the neural network parameter is not a list + s=np.array(s,self.nn.param[0].dtype.name) # convert the state to the same data type as the neural network parameter + else: # if the neural network parameter is a list + s=np.array(s,self.nn.param[0][0].dtype.name) # convert the state to the same data type as the first element of the neural network parameter + if self.episode_step==None: # if episode step value is not given + while True: # loop until episode ends + next_s,r,done,index=self.env(s,epsilon,p,lock,pool_lock) # call the env method witn the state, epsilon value, process index and locks and get the next state, reward, done flag and sampled process index + self.reward[p]+=r # add the reward to the reward array for the process index + s=next_s # assign the next state to the state + if type(self.done_pool[p])==np.ndarray: # if the done flag pool for the process index is a numpy array + self.train_(p,lock,g_lock) # call the train_ method witn the process index and locks to train the neural network model + if done: # if episode ends + if self.PO==1 or self.PO==2: # if the optimization strategy is lock or global lock + lock[1].acquire() # acquire the lock for updating total episode and loss list + self.total_episode.value+=1 # increment total episode by one + self.loss_list.append(self.loss[p]) # append loss array for process index to loss list + if self.PO==1 or self.PO==2: # if optimization strategy is lock or global lock + lock[1].release() # release the lock for updating total episode and loss list + break # break from loop and start next episode + else: # if episode step value is given + for l in range(self.episode_step): # loop through episode step value + next_s,r,done,index=self.env(s,epsilon,p,lock,pool_lock) # call the env method with the state, epsilon value, process index and locks and get the next state, reward, done flag and sampled process index + self.reward[p]+=r # add the reward to the reward array for the process index + s=next_s # assign the next state to the state + if type(self.done_pool[p])==np.ndarray: # if the done flag pool for the process index is a numpy array + self.train_(p,lock,g_lock) # call the train_ method with the process index and locks to train the neural network model + if done: # if episode ends + if self.PO==1 or self.PO==2: # if the optimization strategy is lock or global lock + lock[1].acquire() # acquire the lock for updating total episode and loss list + self.total_episode.value+=1 # increment total episode by one + self.loss_list.append(self.loss[p]) # append loss array for process index to loss list + if self.PO==1 or self.PO==2: # if optimization strategy is lock or global lock + lock[1].release() # release the lock for updating total episode and loss list + break # break from loop and start next episode + if l==self.episode_step-1: # if it is the last step of the episode + if self.PO==1 or self.PO==2: # if the optimization strategy is lock or global lock + lock[1].acquire() # acquire the lock for updating total episode and loss list + self.total_episode.value+=1 # increment total episode by one + self.loss_list.append(self.loss[p]) # append loss array for process index to loss list + if self.PO==1 or self.PO==2: # if optimization strategy is lock or global lock + lock[1].release() # release the lock for updating total episode and loss list + if self.PO==1 or self.PO==2: # if the optimization strategy is lock or global lock + lock[1].acquire() # acquire the lock for updating reward list and reward array + elif len(lock)==3: # if there are three locks + lock[2].acquire() # acquire the third lock for updating reward list and reward array + self.reward_list.append(self.reward[p]) # append the reward array for the process index to the reward list + self.reward[p]=0 # reset the reward array for the process index to zero + if self.PO==1 or self.PO==2: # if the optimization strategy is lock or global lock + lock[1].release() # release the lock for updating reward list and reward array + elif len(lock)==3: # if there are three locks + lock[2].release() # release the third lock for updating reward list and reward array + self.running_flag[p+1]=0 # assign zero value to the running flag list at the position of (process index + 1) to indicate the process is not running + if p not in self.finish_list: # if the process index is not in the finish list + self.finish_list[p]=p # assign the process index to the finish list at the position of process index to indicate the process is finished + if self.PO==1 or self.PO==2: # if the optimization strategy is lock or global lock + lock[1].acquire() # acquire the lock for decrementing process counter + elif self.PO==3: # if the optimization strategy is no lock + lock[1].acquire() # acquire the lock for decrementing process counter + self.process_counter.value-=1 # decrement the process counter by one + if self.PO==1 or self.PO==2: # if the optimization strategy is lock or global lock + lock[1].release() # release the lock for decrementing process counter + elif self.PO==3: # if the optimization strategy is no lock + lock[1].release() # release the lock for decrementing process counter + del self.state_pool[p] # delete the state pool for the process index + del self.action_pool[p] # delete the action pool for the process index + del self.next_state_pool[p] # delete the next state pool for the process index + del self.reward_pool[p] # delete the reward pool for the process index + del self.done_pool[p] # delete the done flag pool for the process index + return # return from the method + + + def stop_func(self): # a method to check if the termination condition is met + if self.end(): # call the end method and check if it returns True + self.save(self.total_episode) # call the save method witn total episode to save the neural network model + self.save_flag.value=True # assign True value to save flag to indicate model is saved + self.stop_flag.value=True # assign True value to stop flag to indicate all processes should stop + return True # return True to indicate termination condition is met + return False # return False to indicate termination condition is not met + + + def stop_func_(self,lock=None): # a method to check if the stop flag is True and release the lock if needed + if self.stop==True: # if the stop attribute is True + if self.stop_flag.value==True or self.stop_func(): # if the stop flag is True or the stop_func method returns True + if self.PO!=3: # if the optimization strategy is not no lock + lock.release() # release the lock + return True # return True to indicate stop condition is met + return False # return False to indicate stop condition is not met + + + def visualize_reward(self): # a method to visualize the reward list as a line plot + print() # print a blank line + plt.figure(1) # create a figure with index 1 + plt.plot(np.arange(len(self.reward_list)),self.reward_list) # plot the reward list as a line with x-axis as the episode index and y-axis as the reward value + plt.xlabel('episode') # set the x-axis label as 'episode' + plt.ylabel('reward') # set the y-axis label as 'reward' + print('reward:{0:.6f}'.format(self.reward_list[-1])) # print the last reward value with six decimal places + return # return from the method + + + def visualize_train(self): # a method to visualize the loss list as a line plot + print() # print a blank line + plt.figure(1) # create a figure with index 1 + plt.plot(np.arange(len(self.loss_list)),self.loss_list) # plot the loss list as a line with x-axis as the episode index and y-axis as the loss value + plt.title('train loss') # set the title of the plot as 'train loss' + plt.xlabel('episode') # set the x-axis label as 'episode' + plt.ylabel('loss') # set the y-axis label as 'loss' + print('loss:{0:.6f}'.format(self.loss_list[-1])) # print the last loss value with six decimal places + return # return from the method + + + def visualize_reward_loss(self): # a method to visualize both the reward list and the loss list as lines on the same plot + print() # print a blank line + plt.figure(1) # create a figure with index 1 + plt.plot(np.arange(len(self.reward_list)),self.reward_list,'r-',label='reward') # plot the reward list as a red line with x-axis as the episode index and y-axis as the reward value and label it as 'reward' + plt.plot(np.arange(len(self.loss_list)),self.loss_list,'b-',label='train loss') # plot the loss list as a blue line witn x-axis as the episode index and y-axis as the loss value and label it as 'train loss' + plt.xlabel('epoch') # set the x-axis label as 'epoch' + plt.ylabel('reward and loss') # set the y-axis label as 'reward and loss' + return # return from the method \ No newline at end of file diff --git a/Note 7.0 documentation/RL/reduced kernel/nspn/kernel_reduced.py b/Note 7.0 documentation/RL/reduced kernel/nspn/kernel_reduced.py new file mode 100644 index 00000000..0b216044 --- /dev/null +++ b/Note 7.0 documentation/RL/reduced kernel/nspn/kernel_reduced.py @@ -0,0 +1,685 @@ +from tensorflow import function +from tensorflow import data as tf_data +import numpy as np +import matplotlib.pyplot as plt +import statistics +import time + + +class kernel: + def __init__(self,nn=None,save_episode=False): + self.nn=nn # the neural network model to be trained + try: + self.nn.km=1 # a flag to indicate that the model is using kernel method + except Exception: + pass + self.platform=None # the platform to use, such as TensorFlow or PyTorch + self.state_pool=None # the pool of states + self.action_pool=None # the pool of actions + self.next_state_pool=None # the pool of next states + self.reward_pool=None # the pool of rewards + self.done_pool=None # the pool of done flags + self.episode_set=[] # the list of episodes + self.epsilon=None # the epsilon value for epsilon-greedy policy + self.episode_step=None # the maximum number of steps per episode + self.pool_size=None # the maximum size of the pool + self.batch=None # the batch size for training + self.update_step=None # the frequency of updating the network parameters + self.trial_count=None # the number of trials to calculate the average reward + self.criterion=None # the criterion for stopping the training + self.reward_list=[] # the list of rewards per episode + self.max_episode_count=None # the maximum number of episodes to save + self.save_episode=save_episode # a flag to indicate whether to save episodes or not + self.loss=None # the loss value for training + self.loss_list=[] # the list of losses per episode + self.sc=0 # a counter for steps + self.total_episode=0 # a counter for episodes + self.time=0 # a timer for training time + self.total_time=0 + + + def action_vec(self): + if self.epsilon!=None: + self.action_one=np.ones(self.action_count,dtype=np.int8) # a vector of ones for action probabilities + return + + + def init(self): + try: + self.nn.pr.TD=np.array(0) # initialize the TD error for prioritized replay buffer + except Exception as e: + try: + if self.nn.pr!=None: + raise e + except Exception: + pass + self.episode_set=[] + self.state_pool=None + self.action_pool=None + self.next_state_pool=None + self.reward_pool=None + self.done_pool=None + self.reward_list=[] + self.loss=0 + self.loss_list=[] + self.sc=0 + self.total_episode=0 + self.time=0 + self.total_time=0 + return + + + def set_up(self,epsilon=None,episode_step=None,pool_size=None,batch=None,update_step=None,trial_count=None,criterion=None): + if epsilon!=None: + self.epsilon=epsilon # set the epsilon value for epsilon-greedy policy + if episode_step!=None: + self.episode_step=episode_step # set the maximum number of steps per episode + if pool_size!=None: + self.pool_size=pool_size # set the maximum size of the pool + if batch!=None: + self.batch=batch # set the batch size for training + if update_step!=None: + self.update_step=update_step # set the frequency of updating the network parameters + if trial_count!=None: + self.trial_count=trial_count # set the number of trials to calculate the average reward + if criterion!=None: + self.criterion=criterion # set the criterion for stopping the training + self.action_vec() # create an action vector with the same length as the number of actions, and fill it with ones to indicate the initial probabilities of each action + return + + + def epsilon_greedy_policy(self,s): + action_prob=self.action_one*self.epsilon/len(self.action_one) # initialize the action probabilities with epsilon + try: + best_a=np.argmax(self.nn.nn.fp(s)) # find the best action according to the network output + action_prob[best_a]+=1-self.epsilon # increase the probability of the best action by 1-epsilon + except Exception as e: + try: + if self.platform.DType!=None: # check if the platform is TensorFlow + raise e + except Exception: + best_a=self.nn.nn(s).argmax() # find the best action according to the network output + action_prob[best_a.numpy()]+=1-self.epsilon # increase the probability of the best action by 1-epsilon + return action_prob # return the action probabilities + + + @function(jit_compile=True) + def tf_opt(self,state_batch,action_batch,next_state_batch,reward_batch,done_batch): + with self.platform.GradientTape(persistent=True) as tape: # create a persistent gradient tape to record the gradients + loss=self.nn.loss(state_batch,action_batch,next_state_batch,reward_batch,done_batch) # calculate the loss function + try: + try: + gradient=self.nn.gradient(tape,loss) # calculate the gradient using the custom gradient function + try: + self.nn.opt.apply_gradients(zip(gradient,self.nn.param)) # apply the gradient to update the network parameters + except Exception: + self.nn.opt(gradient) # apply the gradient to update the network parameters + except Exception: + try: + if self.nn.nn!=None: # check if the network is a single network or a pair of networks + gradient=tape.gradient(loss,self.nn.param) # calculate the gradient using the tape function + self.nn.opt.apply_gradients(zip(gradient,self.nn.param)) # apply the gradient to update the network parameters + except Exception: + actor_gradient=tape.gradient(loss[0],self.nn.param[0]) # calculate the gradient for the actor network + critic_gradient=tape.gradient(loss[1],self.nn.param[1]) # calculate the gradient for the critic network + self.nn.opt.apply_gradients(zip(actor_gradient,self.nn.param[0])) # apply the gradient to update the actor network parameters + self.nn.opt.apply_gradients(zip(critic_gradient,self.nn.param[1])) # apply the gradient to update the critic network parameters + except Exception as e: + raise e + return loss # return the loss value + + + def pytorch_opt(self,state_batch,action_batch,next_state_batch,reward_batch,done_batch): + loss=self.nn.loss(state_batch,action_batch,next_state_batch,reward_batch,done_batch) # calculate the loss function + self.nn.backward(loss) # calculate and accumulate the gradients using the custom backward function + self.nn.opt() # apply the gradients to update the network parameters using the custom opt function + return loss # return the loss value + + + def opt(self,state_batch,action_batch,next_state_batch,reward_batch,done_batch): + try: + if self.platform.DType!=None: # check if the platform is TensorFlow + loss=self.tf_opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch) # use TensorFlow optimization method + except Exception: + loss=self.pytorch_opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch) # use PyTorch optimization method + return loss # return the loss value + + + def pool(self,s,a,next_s,r,done): + if type(self.state_pool)!=np.ndarray and self.state_pool==None: # check if the pool is empty or not + self.state_pool=s # initialize the state pool with s + if type(a)==int: # check if a is an integer or a vector + if type(self.nn.param[0])!=list: + a=np.array(a,self.nn.param[0].dtype.name) # convert a to an array with dtype matching nn.param[0] + else: + a=np.array(a,self.nn.param[0][0].dtype.name) # convert a to an array with dtype matching nn.param[0][0] + self.action_pool=np.expand_dims(a,axis=0) # initialize the action pool with a and add an extra dimension for batch size + else: + if type(self.nn.param[0])!=list: + a=a.astype(self.nn.param[0].dtype.name) # convert a to an array with dtype matching nn.param[0] + else: + a=a.astype(self.nn.param[0][0].dtype.name) # convert a to an array with dtype matching nn.param[0][0] + self.action_poolself.action_pool=a # initialize the action pool with a + self.next_state_pool=np.expand_dims(next_s,axis=0) # initialize the next state pool with next_s and add an extra dimension for batch size + self.reward_pool=np.expand_dims(r,axis=0) # initialize the reward pool with r and add an extra dimension for batch size + self.done_pool=np.expand_dims(done,axis=0) # initialize the done pool with done and add an extra dimension for batch size + else: + self.state_pool=np.concatenate((self.state_pool,s),0) # append s to the state pool along the first axis + if type(a)==int: # check if a is an integer or a vector + if type(self.nn.param[0])!=list: + a=np.array(a,self.nn.param[0].dtype.name) # convert a to an array with dtype matching nn.param[0] + else: + a=np.array(a,self.nn.param[0][0].dtype.name) # convert a to an array with dtype matching nn.param[0][0] + self.action_pool=np.concatenate((self.action_pool,np.expand_dims(a,axis=0)),0) # append a to the action pool along the first axis and add an extra dimension for batch size + else: + if type(self.nn.param[0])!=list: + a=a.astype(self.nn.param[0].dtype.name) # convert a to an array with dtype matching nn.param[0] + else: + a=a.astype(self.nn.param[0][0].dtype.name) # convert a to an array with dtype matching nn.param[0][0] + self.action_pool=np.concatenate((self.action_pool,a),0) # append a to the action pool along the first axis + self.next_state_pool=np.concatenate((self.next_state_pool,np.expand_dims(next_s,axis=0)),0) # append next_s to the next state pool along the first axis and add an extra dimension for batch size + self.reward_pool=np.concatenate((self.reward_pool,np.expand_dims(r,axis=0)),0) # append r to the reward pool along the first axis and add an extra dimension for batch size + self.done_pool=np.concatenate((self.done_pool,np.expand_dims(done,axis=0)),0) # append done to the done pool along the first axis and add an extra dimension for batch size + if len(self.state_pool)>self.pool_size: # check if the pool size exceeds the maximum size + self.state_pool=self.state_pool[1:] # remove the oldest state from the state pool + self.action_pool=self.action_pool[1:] # remove the oldest action from the action pool + self.next_state_pool=self.next_state_pool[1:] # remove the oldest next state from the next state pool + self.reward_pool=self.reward_pool[1:] # remove the oldest reward from the reward pool + self.done_pool=self.done_pool[1:] # remove the oldest done flag from the done pool + return + + + def _train(self): + if len(self.state_pool)self.pool_size: # check if the pool size exceeds the maximum size + TD=np.array(0) # create a zero TD error value + self.nn.pr.TD=np.append(TD,self.nn.pr.TD[2:]) # remove the oldest TD error value from the TD error list and append the zero TD error value + except Exception as e: + try: + if self.nn.pr!=None: # check if the prioritized replay buffer is defined or not + raise e # raise the exception if it is defined + except Exception: + pass + self.reward=r+self.reward # accumulate the reward value over steps + loss=self._train() # train the network using the pool data and get the loss value + self.sc+=1 # increase the step counter by one + if done: # check if done flag is True or not + if self.save_episode==True: # check if save episode flag is True or not + episode=[s,a,next_s,r] # create a list of data for this step + self.reward_list.append(self.reward) # append the reward value to the reward list + return loss,episode,done # return the loss value, episode data, and done flag + elif self.save_episode==True: # check if save episode flag is True or not + episode=[s,a,next_s,r] # create a list of data for this step + s=next_s # update s as next_s for next step + else: + for _ in range(self.episode_step): # loop over the maximum number of steps per episode + try: + try: + if self.platform.DType!=None: # check if the platform is TensorFlow + s=np.expand_dims(s,axis=0) # add an extra dimension for batch size to s + if self.epsilon==None: # check if the epsilon value is defined or not + self.epsilon=self.nn.epsilon(self.sc) # calculate the epsilon value using the custom epsilon function + action_prob=self.epsilon_greedy_policy(s) # get the action probabilities using the epsilon-greedy policy function + a=np.random.choice(self.action_count,p=action_prob) # choose an action randomly according to the action probabilities + except Exception: + s=np.expand_dims(s,axis=0) # add an extra dimension for batch size to s + s=self.platform.tensor(s,dtype=self.platform.float).to(self.nn.device) # convert s to a tensor with dtype matching platform.float and move it to the device + if self.epsilon==None: # check if the epsilon value is defined or not + self.epsilon=self.nn.epsilon(self.sc) # calculate the epsilon value using the custom epsilon function + action_prob=self.epsilon_greedy_policy(s) # get the action probabilities using the epsilon-greedy policy function + a=np.random.choice(self.action_count,p=action_prob) # choose an action randomly according to the action probabilities + next_s,r,done=self.nn.env(a) # get the next state, reward, and done flag from the environment using the custom env function and the chosen action + try: + if self.platform.DType!=None: # check if the platform is TensorFlow + if type(self.nn.param[0])!=list: + next_s=np.array(next_s,self.nn.param[0].dtype.name) # convert next_s to an array with dtype matching nn.param[0] + r=np.array(r,self.nn.param[0].dtype.name) # convert r to an array with dtype matching nn.param[0] + done=np.array(done,self.nn.param[0].dtype.name) # convert done to an array with dtype matching nn.param[0] + else: + next_s=np.array(next_s,self.nn.param[0][0].dtype.name) # convert next_s to an array with dtype matching nn.param[0][0] + r=np.array(r,self.nn.param[0][0].dtype.name) # convert r to an array with dtype matching nn.param[0][0] + done=np.array(done,self.nn.param[0][0].dtype.name) # convert done to an array with dtype matching nn.param[0][0] + except Exception: + pass + except Exception as e: + try: + if self.nn.nn!=None: # check if the network is a single network or a pair of networks + raise e # raise the exception if it is a single network + except Exception: + try: + try: + if self.nn.action!=None: # check if the custom action function is defined or not + try: + if self.platform.DType!=None: # check if the platform is TensorFlow + s=np.expand_dims(s,axis=0) # add an extra dimension for batch size to s + if self.epsilon==None: # check if the epsilon value is defined or not + self.epsilon=self.nn.epsilon(self.sc) # calculate the epsilon value using the custom epsilon function + try: + if self.nn.discriminator!=None: # check if the custom discriminator function is defined or not + a=self.nn.action(s) # get the action using the custom action function + reward=self.nn.discriminator(s,a) # get the reward using the custom discriminator function + s=np.squeeze(s) # remove the extra dimension from s + except Exception: + a=self.nn.action(s).numpy() # get the action using the custom action function and convert it to numpy array + except Exception: + s=np.expand_dims(s,axis=0) # add an extra dimension for batch size to s + s=self.platform.tensor(s,dtype=self.platform.float).to(self.nn.device) # convert s to a tensor with dtype matching platform.float and move it to the device + if self.epsilon==None: # check if the epsilon value is defined or not + self.epsilon=self.nn.epsilon(self.sc) # calculate the epsilon value using the custom epsilon function + try: + if self.nn.discriminator!=None: # check if the custom discriminator function is defined or not + a=self.nn.action(s) # get the action using thecustom action function + reward=self.nn.discriminator(s,a) # get the reward using the custom discriminator function + s=np.squeeze(s) # remove the extra dimension from s + except Exception: + a=self.nn.action(s).detach().numpy() # get the action using the custom action function and convert it to numpy array + except Exception: + try: + if self.platform.DType!=None: # check if the platform is TensorFlow + s=np.expand_dims(s,axis=0) # add an extra dimension for batch size to s + a=(self.nn.actor.fp(s)+self.nn.noise()).numpy() # get the action using the actor network and add some noise and convert it to numpy array + except Exception: + s=np.expand_dims(s,axis=0) # add an extra dimension for batch size to s + s=self.platform.tensor(s,dtype=self.platform.float).to(self.nn.device) # convert s to a tensor with dtype matching platform.float and move it to the device + a=(self.nn.actor(s)+self.nn.noise()).detach().numpy() # get the action using the actor network and add some noise and convert it to numpy array + except Exception as e: + raise e # raise the exception if any error occurs + next_s,r,done=self.nn.env(a) # get the next state, reward, and done flag from the environment using the custom env function and the chosen action + try: + if self.platform.DType!=None: # check if the platform is TensorFlow + if type(self.nn.param[0])!=list: + next_s=np.array(next_s,self.nn.param[0].dtype.name) # convert next_s to an array with dtype matching nn.param[0] + r=np.array(r,self.nn.param[0].dtype.name) # convert r to an array with dtype matching nn.param[0] + done=np.array(done,self.nn.param[0].dtype.name) # convert done to an array with dtype matching nn.param[0] + else: + next_s=np.array(next_s,self.nn.param[0][0].dtype.name) # convert next_s to an array with dtype matching nn.param[0][0] + r=np.array(r,self.nn.param[0][0].dtype.name) # convert r to an array with dtype matching nn.param[0][0] + done=np.array(done,self.nn.param[0][0].dtype.name) # convert done to an array with dtype matching nn.param[0][0] + except Exception: + pass + try: + self.nn.pool(self.state_pool,self.action_pool,self.next_state_pool,self.reward_pool,self.done_pool,[s,a,next_s,reward,done]) # add the data to the pool using the custom pool function + except Exception as e: + try: + if self.nn.pool!=None: # check if the custom pool function is defined or not + raise e # raise the exception if it is defined + except Exception: + self.pool(s,a,next_s,r,done) # add the data to the pool using the default pool function + try: + self.nn.pr.TD=np.append(self.nn.pr.TD,self.nn.initial_TD) # append the initial TD error value to the TD error list for prioritized replay buffer + if len(self.state_pool)>self.pool_size: # check if the pool size exceeds the maximum size + TD=np.array(0) # create a zero TD error value + self.nn.pr.TD=np.append(TD,self.nn.pr.TD[2:]) # remove the oldest TD error value from the TD error list and append the zero TD error value + except Exception as e: + try: + if self.nn.pr!=None: # check if the prioritized replay buffer is defined or not + raise e # raise the exception if it is defined + except Exception: + pass + self.reward=r+self.reward # accumulate the reward value over steps + loss=self._train() # train the network using the pool data and get the loss value + self.sc+=1 # increase the step counter by one + if done: # check if done flag is True or not + if self.save_episode==True: # check if save episode flag is True or not + episode=[s,a,next_s,r] # create a list of data for this step + self.reward_list.append(self.reward) # append the reward value to the reward list + return loss,episode,done # return the loss value, episode data, and done flag + elif self.save_episode==True: # check if save episode flag is True or not + episode=[s,a,next_s,r] # create a list of data for this step + s=next_s # update s as next_s for next step + self.reward_list.append(self.reward) # append the reward value to the reward list + return loss,episode,done # return the loss value, episode data, and done flag + + + def train(self,episode_count,save=None,one=True,p=None,s=None): + avg_reward=None # initialize the average reward value as None + if p==None: # check if p is defined or not + self.p=9 # set p as 9 by default + else: + self.p=p-1 # decrease p by one + if s==None: # check if s is defined or not + self.s=1 # set s as 1 by default + self.file_list=None # set file_list as None by default + else: + self.s=s-1 # decrease s by one + self.file_list=[] # initialize an empty list for file_list + if episode_count!=None: # check if episode_count is defined or not + for i in range(episode_count): # loop over each episode + t1=time.time() # record the start time of the episode + loss,episode,done=self.train_() # train the network for one episode and get the loss value, episode data, and done flag + if self.trial_count!=None: # check if trial_count is defined or not + if len(self.reward_list)>=self.trial_count: # check if the reward list has enough values for trial_count + avg_reward=statistics.mean(self.reward_list[-self.trial_count:]) # calculate the average reward value using the last trial_count values in the reward list + if self.criterion!=None and avg_reward>=self.criterion: # check if criterion is defined or not and if the average reward value meets the criterion or not + t2=time.time() # record the end time of the episode + self.total_time+=(t2-t1) # accumulate the total time over episodes + self._time=self.total_time-int(self.total_time) # get the decimal part of the total time + if self._time<0.5: + self.total_time=int(self.total_time) # round down the total time if the decimal part is less than 0.5 + else: + self.total_time=int(self.total_time)+1 # round up the total time if the decimal part is greater than or equal to 0.5 + print('episode:{0}'.format(self.total_episode)) # print the total number of episodes + print('last loss:{0:.6f}'.format(loss)) # print the last loss value + print('average reward:{0}'.format(avg_reward)) # print the average reward value + print() + print('time:{0}s'.format(self.total_time)) # print the total time in seconds + return # stop training and return + self.loss=loss # update loss as the last loss value + self.loss_list.append(loss) # append loss to the loss list + self.total_episode+=1 # increase the total episode counter by one + if episode_count%10!=0: + p=episode_count-episode_count%self.p + p=int(p/self.p) + s=episode_count-episode_count%self.s + s=int(s/self.s) + else: + p=episode_count/(self.p+1) + p=int(p) + s=episode_count/(self.s+1) + s=int(s) + if p==0: + p=1 + if s==0: + s=1 + if i%p==0: # check if i is a multiple of p or not + if len(self.state_pool)>=self.batch: # check if the pool size is larger than or equal to the batch size or not + print('episode:{0} loss:{1:.6f}'.format(i+1,loss)) # print the episode number and loss value + if avg_reward!=None: # check if avg_reward is None or not + print('episode:{0} average reward:{1}'.format(i+1,avg_reward)) # print the episode number and average reward value + else: + print('episode:{0} reward:{1}'.format(i+1,self.reward)) # print the episode number and reward value + print() + if save!=None and i%s==0: # check if save is None or not and if i is a multiple of s orif save!=None and i%s==0: # check if save is None or not and if i is a multiple of s or not + self.save(self.total_episode,one) # save the network parameters and training data using the custom save function and the total episode number and one flag + if self.save_episode==True: # check if save episode flag is True or not + if done: # check if done flag is True or not + episode.append('done') # append 'done' to the episode data list + self.episode_set.append(episode) # append the episode data list to the episode set list + if self.max_episode_count!=None and len(self.episode_set)>=self.max_episode_count: # check if max episode count is defined or not and if the episode set size reaches the max episode count or not + self.save_episode=False # set the save episode flag as False to stop saving episodes + try: + try: + self.nn.ec.assign_add(1) # increase the episode counter by one using TensorFlow assign_add method + except Exception: + self.nn.ec+=1 # increase the episode counter by one using Python addition operator + except Exception: + pass + t2=time.time() # record the end time of the episode + self.time+=(t2-t1) # accumulate the time over episodes + else: + i=0 # initialize the episode index as zero + while True: # loop until an exception occurs or the criterion is met + t1=time.time() # record the start time of the episode + loss,episode,done=self.train_() # train the network for one episode and get the loss value, episode data, and done flag + if self.trial_count!=None: # check if trial_count is defined or not + if len(self.reward_list)==self.trial_count: # check if the reward list has enough values for trial_count + avg_reward=statistics.mean(self.reward_list[-self.trial_count:]) # calculate the average reward value using the last trial_count values in the reward list + if avg_reward>=self.criterion: # check if the average reward value meets the criterion or not + t2=time.time() # record the end time of the episode + self.total_time+=(t2-t1) # accumulate the total time over episodes + self._time=self.total_time-int(self.total_time) # get the decimal part of the total time + if self._time<0.5: + self.total_time=int(self.total_time) # round down the total time if the decimal part is less than 0.5 + else: + self.total_time=int(self.total_time)+1 # round up the total time if the decimal part is greater than or equal to 0.5 + print('episode:{0}'.format(self.total_episode)) # print the total number of episodes + print('last loss:{0:.6f}'.format(loss)) # print the last loss value + print('average reward:{0}'.format(avg_reward)) # print the average reward value + print() + print('time:{0}s'.format(self.total_time)) # print the total time in seconds + return # stop training and return + self.loss=loss # update loss as the last loss value + self.loss_list.append(loss) # append loss to the loss list + i+=1 # increase the episode index by one + self.total_episode+=1 # increase the total episode counter by one + if episode_count%10!=0: + p=episode_count-episode_count%self.p + p=int(p/self.p) + s=episode_count-episode_count%self.s + s=int(s/self.s) + else: + p=episode_count/(self.p+1) + p=int(p) + s=episode_count/(self.s+1) + s=int(s) + if p==0: + p=1 + if s==0: + s=1 + if i%p==0: # check if i is a multiple of p or not + if len(self.state_pool)>=self.batch: # check if the pool size is larger than or equal to the batch size or not + print('episode:{0} loss:{1:.6f}'.format(i+1,loss)) # print the episode number and loss value + if avg_reward!=None: # check if avg_reward is None or not + print('episode:{0} average reward:{1}'.format(i+1,avg_reward)) # print the episode number and average reward valueelse: + else: + print('episode:{0} reward:{1}'.format(i+1,self.reward)) # print the episode number and reward value + print() + if save!=None and i%s==0: # check if save is None or not and if i is a multiple of s or not + self.save(self.total_episode,one) # save the network parameters and training data using the custom save function and the total episode number and one flag + if self.save_episode==True: # check if save episode flag is True or not + if done: # check if done flag is True or not + episode.append('done') # append 'done' to the episode data list + self.episode_set.append(episode) # append the episode data list to the episode set list + if self.max_episode_count!=None and len(self.episode_set)>=self.max_episode_count: # check if max episode count is defined or not and if the episode set size reaches the max episode count or not + self.save_episode=False # set the save episode flag as False to stop saving episodes + try: + try: + self.nn.ec.assign_add(1) # increase the episode counter by one using TensorFlow assign_add method + except Exception: + self.nn.ec+=1 # increase the episode counter by one using Python addition operator + except Exception: + pass + t2=time.time() # record the end time of the episode + self.time+=(t2-t1) # accumulate the time over episodes + self._time=self.time-int(self.time) # get the decimal part of the time + if self._time<0.5: + self.total_time=int(self.time) # round down the time if the decimal part is less than 0.5 + else: + self.total_time=int(self.time)+1 # round up the time if the decimal part is greater than or equal to 0.5 + self.total_time+=self.time # add the time to the total time + print('last loss:{0:.6f}'.format(loss)) # print the last loss value + print('last reward:{0}'.format(self.reward)) # print the last reward value + print() + print('time:{0}s'.format(self.time)) # print the time in seconds + return # return + + + def visualize_reward(self): + print() + plt.figure(1) # create a figure object with number 1 + plt.plot(np.arange(self.total_episode),self.reward_list) # plot the reward list against the episode number + plt.xlabel('episode') # set the x-axis label as 'episode' + plt.ylabel('reward') # set the y-axis label as 'reward' + print('reward:{0:.6f}'.format(self.reward_list[-1])) # print the last reward value + return + + + def visualize_train(self): + print() + plt.figure(1) # create a figure object with number 1 + plt.plot(np.arange(self.total_episode),self.loss_list) # plot the loss list against the episode number + plt.title('train loss') # set the title of the figure as 'train loss' + plt.xlabel('episode') # set the x-axis label as 'episode' + plt.ylabel('loss') # set the y-axis label as 'loss' + print('loss:{0:.6f}'.format(self.loss_list[-1])) # print the last loss value + return \ No newline at end of file diff --git a/Note 7.0 documentation/compiler/nc.txt b/Note 7.0 documentation/compiler/nc.txt new file mode 100644 index 00000000..14ea9e06 --- /dev/null +++ b/Note 7.0 documentation/compiler/nc.txt @@ -0,0 +1,33 @@ +Note Compiler + +Note Compiler can compile filename.n into filename.py. +Using Note compiler can use different grammar to write your neural network in filename.n. + +Grammar transition by Note Compiler: +Operator: +(object.*object)->tf.matmul(object1,object2) +(object.^)->tf.reverse(object) +(object1.|object2)->tf.concat([object1,object2],0) +(object1.||object2)->tf.concat([object1,object2],1) +(object./)->tf.math.sqrt(object) +(object1.=object2)->state_ops.assign(object1,object2) + +Init: +example: +param=zint32. [3,4]->param=tf.zeros([3,4],dtype=tf.int32) +return zint32. [3,4]->return tf.zeros([3,4],dtype=tf.int32) +param=uint32. (0,1)[3,4]->param=tf.random.uniform([3,4],0,1,dtype=tf.int32) +return uint32. (0,1)[3,4]->return tf.random.uniform([3,4],0,1,dtype=tf.int32) +param=nint32. (0,1)[3,4]->param=tf.random.normal([3,4],0,1,dtype=tf.int32) +return nint32. (0,1)[3,4]->return tf.random.normal([3,4],0,1,dtype=tf.int32) +param=oint32. [3,4]->param=tf.ones([3,4],dtype=tf.int32) +return oint32. [3,4]->return tf.ones([3,4],dtype=tf.int32) + +Using example: +import Note.create.nc as nc +c=nc.compiler('nn.n') +c.Compile() +#then you will get filename.py. + +Macro: +define. macro name string \ No newline at end of file diff --git a/Note 7.0 documentation/compiler/nn.n b/Note 7.0 documentation/compiler/nn.n new file mode 100644 index 00000000..fdf619f1 --- /dev/null +++ b/Note 7.0 documentation/compiler/nn.n @@ -0,0 +1,26 @@ +import tensorflow as tf +define. Pi 3.14 + +class nn: #a simple example,a neural network class + def __init__(self): + self.weight1=tf.Variable(nfloat32. [784,64]) + self.bias1=tf.Variable(nfloat32. [64]) + self.weight2=tf.Variable(nfloat32. [64,64]) + self.bias2=tf.Variable(nfloat32. [64]) + self.weight3=tf.Variable(nfloat32. [64,10]) + self.bias3=tf.Variable(nfloat32. [10]) + self.param=[self.weight1,self.weight2,self.weight3,self.bias1,self.bias2,self.bias3] + self.opt=tf.keras.optimizers.Adam() + self.info='example' + + + def fp(self,data): + with tf.device('GPU:0'): + layer1=tf.nn.relu((data.*self.weight1)+self.bias1) + layer2=tf.nn.relu((layer1.*self.weight2)+self.bias2) + output=(layer2.*self.weight3)+self.bias3+Pi + return output + + + def loss(self,output,labels): + return tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=output,labels=labels)) \ No newline at end of file diff --git a/Note 7.0 documentation/compiler/nn.py b/Note 7.0 documentation/compiler/nn.py new file mode 100644 index 00000000..7d02bda4 --- /dev/null +++ b/Note 7.0 documentation/compiler/nn.py @@ -0,0 +1,25 @@ +import tensorflow as tf + +class nn: #a simple example,a neural network class + def __init__(self): + self.weight1=tf.Variable(tf.random.normal([784,64],dtype=tf.float32)) + self.bias1=tf.Variable(tf.random.normal([64],dtype=tf.float32)) + self.weight2=tf.Variable(tf.random.normal([64,64],dtype=tf.float32)) + self.bias2=tf.Variable(tf.random.normal([64],dtype=tf.float32)) + self.weight3=tf.Variable(tf.random.normal([64,10],dtype=tf.float32)) + self.bias3=tf.Variable(tf.random.normal([10],dtype=tf.float32)) + self.param=[self.weight1,self.weight2,self.weight3,self.bias1,self.bias2,self.bias3] + self.opt=tf.keras.optimizers.Adam() + self.info='example' + + + def fp(self,data): + with tf.device('GPU:0'): + layer1=tf.nn.relu(tf.matmul(data,self.weight1)+self.bias1) + layer2=tf.nn.relu(tf.matmul(layer1,self.weight2)+self.bias2) + output=tf.matmul(layer2,self.weight3)+self.bias3+3.14 + return output + + + def loss(self,output,labels): + return tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=output,labels=labels)) \ No newline at end of file From 48f4ed6733870671aa4ce6d86bc6bff1215ecdef Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Sun, 9 Jul 2023 11:33:11 +0800 Subject: [PATCH 037/337] Update README.md --- README.md | 63 ++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 44 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 32f05f19..692b01df 100644 --- a/README.md +++ b/README.md @@ -114,7 +114,7 @@ for p in range(7): kernel.update_nn_param() kernel.test(x_train,y_train,32) ``` -### Stop training and saving when condition is met: +### PO3： ```python import Note.DL.process.kernel as k #import kernel import tensorflow as tf @@ -123,26 +123,23 @@ from multiprocessing import Process,Lock,Manager mnist=tf.keras.datasets.mnist (x_train,y_train),(x_test,y_test)=mnist.load_data() x_train,x_test =x_train/255.0,x_test/255.0 -x_train=x_train.reshape([60000,784]) nn=n.nn() #create neural network object nn.build() kernel=k.kernel(nn) #start kernel -kernel.stop=True -kernel.end_loss=0.7 #stop condition -kernel.process=7 #7 processes to train +kernel.process=7 #7 processes to train kernel.data_segment_flag=True kernel.epoch=6 #epoch:6 kernel.batch=32 #batch:32 -kernel.PO=2 #use PO3 +kernel.PO=3 #use PO4 kernel.data(x_train,y_train) #input train data manager=Manager() #create manager object kernel.init(manager) #initialize shared data -lock=[Lock(),Lock()] -g_lock=Lock() -for p in range(7): - Process(target=kernel.train,args=(p,lock,g_lock)).start() +lock=Lock() +for p in range(7): + Process(target=kernel.train,args=(p,lock)).start() +kernel.update_nn_param() +kernel.test(x_train,y_train,32) ``` -### PO3： ```python import Note.DL.process.kernel as k #import kernel import tensorflow as tf @@ -162,12 +159,12 @@ kernel.PO=3 #use PO4 kernel.data(x_train,y_train) #input train data manager=Manager() #create manager object kernel.init(manager) #initialize shared data -lock=Lock() for p in range(7): - Process(target=kernel.train,args=(p,lock)).start() + Process(target=kernel.train,args=(p,)).start() kernel.update_nn_param() kernel.test(x_train,y_train,32) ``` +### Parallel test: ```python import Note.DL.process.kernel as k #import kernel import tensorflow as tf @@ -179,18 +176,46 @@ x_train,x_test =x_train/255.0,x_test/255.0 nn=n.nn() #create neural network object nn.build() kernel=k.kernel(nn) #start kernel -kernel.process=7 #7 processes to train +kernel.process=7 #7 processes to train +kernel.process_t=3 kernel.data_segment_flag=True kernel.epoch=6 #epoch:6 kernel.batch=32 #batch:32 -kernel.PO=3 #use PO4 +kernel.PO=1 #use PO1 +kernel.data(x_train,y_train,x_test,y_test) #input train data +manager=Manager() #create manager object +kernel.init(manager) #initialize shared data +lock=[Lock(),Lock()] +for p in range(7): + Process(target=kernel.train,args=(p,lock,None,32)).start() +``` +### Stop training and saving when condition is met: +```python +import Note.DL.process.kernel as k #import kernel +import tensorflow as tf +import nn as n #import neural network +from multiprocessing import Process,Lock,Manager +mnist=tf.keras.datasets.mnist +(x_train,y_train),(x_test,y_test)=mnist.load_data() +x_train,x_test =x_train/255.0,x_test/255.0 +x_train=x_train.reshape([60000,784]) +nn=n.nn() #create neural network object +nn.build() +kernel=k.kernel(nn) #start kernel +kernel.stop=True +kernel.end_loss=0.7 #stop condition +kernel.process=7 #7 processes to train +kernel.data_segment_flag=True +kernel.epoch=6 #epoch:6 +kernel.batch=32 #batch:32 +kernel.PO=2 #use PO3 kernel.data(x_train,y_train) #input train data manager=Manager() #create manager object kernel.init(manager) #initialize shared data -for p in range(7): - Process(target=kernel.train,args=(p,)).start() -kernel.update_nn_param() -kernel.test(x_train,y_train,32) +lock=[Lock(),Lock()] +g_lock=Lock() +for p in range(7): + Process(target=kernel.train,args=(p,lock,g_lock)).start() ``` From 64ffba30d32f595c389591153b76eca27532a16c Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Sun, 9 Jul 2023 12:05:12 +0800 Subject: [PATCH 038/337] Update README.md --- README.md | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 692b01df..7247a4e4 100644 --- a/README.md +++ b/README.md @@ -38,8 +38,24 @@ kernel.visualize_reward() # Parallel test: **You can download neural network example in this link,and then you can import neural network and train with kernel,link and example code are below.** -https://github.com/NoteDancing/Note-documentation/blob/Note-7.0-pv/Note%207.0%20pv%20documentation/DL/neural%20network/tensorflow/process/nn.py +https://github.com/NoteDancing/Note-documentation/blob/Note-7.0/Note%207.0%20documentation/DL/neural%20network/tensorflow/process/nn.py +```python +import Note.DL.kernel as k #import kernel +import tensorflow as tf #import platform +import nn as n #import neural network +mnist=tf.keras.datasets.mnist +(x_train,y_train),(x_test,y_test)=mnist.load_data() +x_train,x_test =x_train/255.0,x_test/255.0 +nn=n.nn() #create neural network object +kernel=k.kernel(nn) #start kernel +kernel.platform=tf #use platform +kernel.process_t=3 +kernel.data(x_train,y_train,x_test,y_test) #input train data +kernel.train(32,5,32) #train neural network + #batch size:32 + #epoch:5 +``` ```python import tensorflow as tf import nn_acc as n @@ -223,7 +239,7 @@ for p in range(7): ### Pool Network: **You can download neural network example in this link,and then you can import neural network and train with kernel,link and example code are below.** -https://github.com/NoteDancing/Note-documentation/blob/Note-7.0-pv/Note%207.0%20pv%20documentation/RL/neural%20network/tensorflow/pool%20net/DQN.py +https://github.com/NoteDancing/Note-documentation/blob/Note-7.0/Note%207.0%20documentation/RL/neural%20network/tensorflow/pool%20net/DQN.py #### PO2: ```python @@ -284,7 +300,7 @@ for p in range(5): # Online training: **You can download neural network example in this link,and then you can import neural network and train with kernel,link and example code are below.** -https://github.com/NoteDancing/Note-documentation/blob/Note-7.0-pv/Note%207.0%20pv%20documentation/DL/neural%20network/tensorflow/nn_ol.py +https://github.com/NoteDancing/Note-documentation/blob/Note-7.0/Note%207.0%20documentation/DL/neural%20network/tensorflow/nn_ol.py **example:** ```python @@ -327,7 +343,7 @@ t.test(dqn,tf,2) # Note Compiler: -documentation:https://github.com/NoteDancing/Note-documentation/tree/Note-7.0-pv/Note%207.0%20pv%20documentation/compiler +documentation:https://github.com/NoteDancing/Note-documentation/tree/Note-7.0/Note%207.0%20documentation/compiler ```python import Note.nc as nc c=nc.compiler('nn.n') From 1313a665572168fd57a84198da838170c62b8546 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Sun, 9 Jul 2023 13:55:33 +0800 Subject: [PATCH 039/337] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 7247a4e4..8bf57ca9 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,7 @@ mnist=tf.keras.datasets.mnist (x_train,y_train),(x_test,y_test)=mnist.load_data() x_train,x_test =x_train/255.0,x_test/255.0 nn=n.nn() #create neural network object +nn.build() kernel=k.kernel(nn) #start kernel kernel.platform=tf #use platform kernel.process_t=3 From 1a7149b2f9e0d7250bffe56260139ba7ef39ba81 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Mon, 10 Jul 2023 19:16:07 +0800 Subject: [PATCH 040/337] Update kernel_reduced.py --- .../DL/reduced kernel/kernel_reduced.py | 46 ++++--------------- 1 file changed, 9 insertions(+), 37 deletions(-) diff --git a/Note 7.0 documentation/DL/reduced kernel/kernel_reduced.py b/Note 7.0 documentation/DL/reduced kernel/kernel_reduced.py index 392d5897..1ff522be 100644 --- a/Note 7.0 documentation/DL/reduced kernel/kernel_reduced.py +++ b/Note 7.0 documentation/DL/reduced kernel/kernel_reduced.py @@ -562,25 +562,13 @@ def test(self,test_data=None,test_labels=None,batch=None): # a method to calcul else: # if a numpy array is used for test data total_loss=0 # initialize the total loss value for all batches total_acc=0 # initialize the total accuracy value for all batches - if type(test_data)==list: # if test data is a list of arrays (for multiple inputs) - batches=int((test_data[0].shape[0]-test_data[0].shape[0]%batch)/batch) # calculate how many batches are needed for test data according to batch size - shape0=test_data[0].shape[0] # get the number of samples in test data - else: # if test data is a single array - batches=int((test_data.shape[0]-test_data.shape[0]%batch)/batch) # calculate how many batches are needed for test data according to batch size - shape0=test_data.shape[0] # get the number of samples in test data + batches=int((test_data.shape[0]-test_data.shape[0]%batch)/batch) # calculate how many batches are needed for test data according to batch size + shape0=test_data.shape[0] # get the number of samples in test data for j in range(batches): # iterate over each batch index index1=j*batch # calculate the start index of test data for this batch index2=(j+1)*batch # calculate the end index of test data for this batch - if type(test_data)==list: # if test data is a list of arrays (for multiple inputs) - for i in range(len(test_data)): - data_batch[i]=test_data[i][index1:index2] # get a slice of test data according to the index range for each input array - else: # if test data is a single array - data_batch=test_data[index1:index2] # get a slice of test data according to the index range - if type(test_labels)==list: # if test labels is a list of arrays (for multiple outputs) - for i in range(len(test_labels)): - labels_batch[i]=test_labels[i][index1:index2] # get a slice of test labels according to the index range for each output array - else: # if test labels is a single array - labels_batch=test_labels[index1:index2] # get a slice of test labels according to the index range + data_batch=test_data[index1:index2] # get a slice of test data according to the index range + labels_batch=test_labels[index1:index2] # get a slice of test labels according to the index range output=self.nn.fp(data_batch) # get the output value using the neural network's forward propagation method batch_loss=self.nn.loss(output,labels_batch) # get the batch loss value using the neural network's loss function total_loss+=batch_loss # accumulate the batch loss to total loss @@ -599,27 +587,11 @@ def test(self,test_data=None,test_labels=None,batch=None): # a method to calcul index2=batch-(shape0-batches*batch) # calculate how many samples are needed from the beginning of test data to form a full batch try: try: - if type(test_data)==list: # if test data is a list of arrays (for multiple if type(test_data)==list: # if test data is a list of arrays (for multiple inputs) - for i in range(len(test_data)): - data_batch[i]=self.platform.concat([test_data[i][index1:],test_data[i][:index2]],0) # concatenate two slices of test data from the end and the beginning to form a batch for each input array - else: # if test data is a single array - data_batch=self.platform.concat([test_data[index1:],test_data[:index2]],0) # concatenate two slices of test data from the end and the beginning to form a batch - if type(test_labels)==list: # if test labels is a list of arrays (for multiple outputs) - for i in range(len(test_labels)): - labels_batch[i]=self.platform.concat([test_labels[i][index1:],test_labels[i][:index2]],0) # concatenate two slices of test labels from the end and the beginning to form a batch for each output array - else: # if test labels is a single array - labels_batch=self.platform.concat([test_labels[index1:],test_labels[:index2]],0) # concatenate two slices of test labels from the end and the beginning to form a batch + data_batch=self.platform.concat([test_data[index1:],test_data[:index2]],0) # concatenate two slices of test data from the end and the beginning to form a batch + labels_batch=self.platform.concat([test_labels[index1:],test_labels[:index2]],0) # concatenate two slices of test labels from the end and the beginning to form a batch except Exception: # if the platform's concat method fails - if type(test_data)==list: # if test data is a list of arrays (for multiple inputs) - for i in range(len(test_data)): - data_batch[i]=np.concatenate([test_data[i][index1:],test_data[i][:index2]],0) # use numpy's concatenate method instead for each input array - else: # if test data is a single array - data_batch=np.concatenate([test_data[index1:],test_data[:index2]],0) # use numpy's concatenate method instead - if type(test_labels)==list: # if test labels is a list of arrays (for multiple outputs) - for i in range(len(test_labels)): - labels_batch[i]=np.concatenate([test_labels[i][index1:],test_labels[i][:index2]],0) # use numpy's concatenate method instead for each output array - else: # if test labels is a single array - labels_batch=np.concatenate([test_labels[index1:],test_labels[:index2]],0) # use numpy's concatenate method instead + data_batch=np.concatenate([test_data[index1:],test_data[:index2]],0) # use numpy's concatenate method instead + labels_batch=np.concatenate([test_labels[index1:],test_labels[:index2]],0) # use numpy's concatenate method instead except Exception as e: raise e # raise any other exception output=self.nn.fp(data_batch) # get the output value using the neural network's forward propagation method @@ -738,4 +710,4 @@ def visualize_train(self): # a method to visualize the train loss and accuracy print('train acc:{0:.6f}'.format(self.train_acc)) # print the last train accuracy value in decimal except Exception: # if the neural network does not have an accuracy method defined or it fails pass - return # return nothing for this case \ No newline at end of file + return # return nothing for this case From 925e7e960c31a0016597af8dcf8a030830bca9a9 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Mon, 10 Jul 2023 19:16:22 +0800 Subject: [PATCH 041/337] Update kernel_reduced.py --- .../DL/reduced kernel/process/kernel_reduced.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/Note 7.0 documentation/DL/reduced kernel/process/kernel_reduced.py b/Note 7.0 documentation/DL/reduced kernel/process/kernel_reduced.py index 41528280..54629697 100644 --- a/Note 7.0 documentation/DL/reduced kernel/process/kernel_reduced.py +++ b/Note 7.0 documentation/DL/reduced kernel/process/kernel_reduced.py @@ -516,11 +516,7 @@ def test(self,test_data=None,test_labels=None,batch=None,p=None): index1=j*batch # get the start index of the batch index2=(j+1)*batch # get the end index of the batch data_batch=test_data[index1:index2] # get the data batch from test data array - if type(test_labels)==list: # check if test labels is a list - for i in range(len(test_labels)): - labels_batch[i]=test_labels[i][index1:index2] # get the labels batch from test labels list - else: - labels_batch=test_labels[index1:index2] # get the labels batch from test labels array + labels_batch=test_labels[index1:index2] # get the labels batch from test labels array try: try: output=self.nn.fp(data_batch) # use the neural network's forward propagation function to get the output @@ -634,4 +630,4 @@ def visualize_train(self): print('train acc:{0:.6f}'.format(self.train_acc.value)) # print the train accuracy value with six decimal places except Exception: pass - return \ No newline at end of file + return From c4ad2d0962318ed195ab354f3056ef83c975e4c4 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Wed, 12 Jul 2023 13:59:23 +0800 Subject: [PATCH 042/337] Update nn_ol.py --- .../DL/neural network/tensorflow/nn_ol.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/nn_ol.py b/Note 7.0 documentation/DL/neural network/tensorflow/nn_ol.py index c32bc542..0f3150e0 100644 --- a/Note 7.0 documentation/DL/neural network/tensorflow/nn_ol.py +++ b/Note 7.0 documentation/DL/neural network/tensorflow/nn_ol.py @@ -16,13 +16,13 @@ def __init__(self,data,labels): self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) self.opt=tf.keras.optimizers.Adam() #optimizer,kernel uses it to optimize. self.train_loss_list=[] - self.c=0 #counter + self.counter=0 #counter self.max_length=1000 - def ol(self): #This is simulative online function. + def online(self): #This is simulative online function. index=np.random.choice(60000,size=[32]) - if self.c==10000: + if self.counter==10000: return 'stop' else: return [self.train_data[index],self.train_labels[index]] @@ -34,4 +34,4 @@ def fp(self,data): #forward propagation function,kernel uses it for forward pro def loss(self,output,labels): #loss functino,kernel uses it to calculate loss. - return self.loss_object(labels,output) \ No newline at end of file + return self.loss_object(labels,output) From 7b6c2824656b29a3d77427ca3cb71f196e90a8d5 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 13 Jul 2023 13:45:17 +0800 Subject: [PATCH 043/337] Update and rename kernel_reduced.py to kernel.py --- .../DL/{reduced kernel/kernel_reduced.py => kernel/kernel.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename Note 7.0 documentation/DL/{reduced kernel/kernel_reduced.py => kernel/kernel.py} (100%) diff --git a/Note 7.0 documentation/DL/reduced kernel/kernel_reduced.py b/Note 7.0 documentation/DL/kernel/kernel.py similarity index 100% rename from Note 7.0 documentation/DL/reduced kernel/kernel_reduced.py rename to Note 7.0 documentation/DL/kernel/kernel.py From 02c4e09a3c6113f8af28a02a146ae17d45474d35 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 13 Jul 2023 13:45:44 +0800 Subject: [PATCH 044/337] Update and rename kernel_reduced.py to kernel.py --- .../process/kernel_reduced.py => kernel/process/kernel.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename Note 7.0 documentation/DL/{reduced kernel/process/kernel_reduced.py => kernel/process/kernel.py} (100%) diff --git a/Note 7.0 documentation/DL/reduced kernel/process/kernel_reduced.py b/Note 7.0 documentation/DL/kernel/process/kernel.py similarity index 100% rename from Note 7.0 documentation/DL/reduced kernel/process/kernel_reduced.py rename to Note 7.0 documentation/DL/kernel/process/kernel.py From fa4aef1a639697fc276ccc9299e682baf287ffb9 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 13 Jul 2023 13:46:04 +0800 Subject: [PATCH 045/337] Update and rename kernel_reduced.py to kernel.py --- .../RL/{reduced kernel/kernel_reduced.py => kernel/kernel.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename Note 7.0 documentation/RL/{reduced kernel/kernel_reduced.py => kernel/kernel.py} (98%) diff --git a/Note 7.0 documentation/RL/reduced kernel/kernel_reduced.py b/Note 7.0 documentation/RL/kernel/kernel.py similarity index 98% rename from Note 7.0 documentation/RL/reduced kernel/kernel_reduced.py rename to Note 7.0 documentation/RL/kernel/kernel.py index 25425e0c..0c7a3c74 100644 --- a/Note 7.0 documentation/RL/reduced kernel/kernel_reduced.py +++ b/Note 7.0 documentation/RL/kernel/kernel.py @@ -667,4 +667,4 @@ def visualize_reward_loss(self): # a method to visualize both the reward list a plt.plot(np.arange(len(self.loss_list)),self.loss_list,'b-',label='train loss') # plot the loss list as a blue line witn x-axis as the episode index and y-axis as the loss value and label it as 'train loss' plt.xlabel('epoch') # set the x-axis label as 'epoch' plt.ylabel('reward and loss') # set the y-axis label as 'reward and loss' - return # return from the method \ No newline at end of file + return # return from the method From d03599aa98718d354f71062c4d08bb7be03726f7 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 13 Jul 2023 13:47:11 +0800 Subject: [PATCH 046/337] Update and rename kernel_reduced.py to kernel.py --- .../nspn/kernel_reduced.py => kernel/nspn/kernel.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename Note 7.0 documentation/RL/{reduced kernel/nspn/kernel_reduced.py => kernel/nspn/kernel.py} (98%) diff --git a/Note 7.0 documentation/RL/reduced kernel/nspn/kernel_reduced.py b/Note 7.0 documentation/RL/kernel/nspn/kernel.py similarity index 98% rename from Note 7.0 documentation/RL/reduced kernel/nspn/kernel_reduced.py rename to Note 7.0 documentation/RL/kernel/nspn/kernel.py index 0b216044..0ddd98a9 100644 --- a/Note 7.0 documentation/RL/reduced kernel/nspn/kernel_reduced.py +++ b/Note 7.0 documentation/RL/kernel/nspn/kernel.py @@ -682,4 +682,4 @@ def visualize_train(self): plt.xlabel('episode') # set the x-axis label as 'episode' plt.ylabel('loss') # set the y-axis label as 'loss' print('loss:{0:.6f}'.format(self.loss_list[-1])) # print the last loss value - return \ No newline at end of file + return From 5cc83c12566ac9de2cce895bac4bcb1de3b225d3 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Sat, 15 Jul 2023 09:06:25 +0800 Subject: [PATCH 047/337] Update README.md --- README.md | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 8bf57ca9..07a2c04e 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,26 @@ kernel.visualize_reward() ``` +# Training with test data +```python +import Note.DL.kernel as k #import kernel +import tensorflow as tf #import platform +import nn_acc as n #import neural network +mnist=tf.keras.datasets.mnist +(x_train,y_train),(x_test,y_test)=mnist.load_data() +x_train,x_test =x_train/255.0,x_test/255.0 +nn=n.nn() #create neural network object +kernel=k.kernel(nn) #start kernel +kernel.platform=tf #use platform +kernel.data(x_train,y_train,x_test,y_test) #input train data +kernel.train(32,5,32) #train neural network + #batch size:32 + #test batch size:32 + #epoch:5 +kernel.save() #save neural network +``` + + # Parallel test: **You can download neural network example in this link,and then you can import neural network and train with kernel,link and example code are below.** @@ -318,28 +338,28 @@ kernel.train_ol() #train neural network ``` -# Test neural network: +# Check neural network: ## DL: You can test it before using the kernel training neural network. ```python -import Note.DL.dl.test_nn as t +from Note.DL.dl.check_nn import check import tensorflow as tf #import platform import nn as n #import neural network mnist=tf.keras.datasets.mnist (x_train,y_train),(x_test,y_test)=mnist.load_data() x_train,x_test =x_train/255.0,x_test/255.0 nn=n.nn() -t.test(cnn,tf,x_train[:32],y_train[:32]) +check(cnn,tf,x_train[:32],y_train[:32]) ``` ## RL: You can test it before using the kernel training neural network. ```python -import Note.RL.rl.test_nn as t +from Note.RL.rl.check_nn import check import tensorflow as tf #import platform import DQN as d dqn=d.DQN(4,128,2) #create neural network object -t.test(dqn,tf,2) +checkdqn,tf,2) ``` From 137583697cd95f01a5cc4618150e4778ac183e07 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Sat, 15 Jul 2023 09:09:01 +0800 Subject: [PATCH 048/337] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 07a2c04e..5375e96e 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,7 @@ kernel.visualize_reward() # Training with test data +https://github.com/NoteDancing/Note-documentation/blob/Note-7.0/Note%207.0%20documentation/DL/neural%20network/tensorflow/nn_acc.py ```python import Note.DL.kernel as k #import kernel import tensorflow as tf #import platform From dc0ba508a687bfb90fe46435ed4a8d2aee1dbf43 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Sat, 15 Jul 2023 09:09:53 +0800 Subject: [PATCH 049/337] Update kernel.txt --- Note 7.0 documentation/RL/kernel.txt | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Note 7.0 documentation/RL/kernel.txt b/Note 7.0 documentation/RL/kernel.txt index 321998a2..8b6a9b06 100644 --- a/Note 7.0 documentation/RL/kernel.txt +++ b/Note 7.0 documentation/RL/kernel.txt @@ -2,9 +2,3 @@ Pool network: Pool network use multiprocessing parallel and random add episode in pool,which would make data being uncorrelated in pool, then pools would be used parallel training agent. Each process have a pool. - -Support stop mechanism. -Support prioritized replay. -Support gradient attenuation. -Support memory protection mechanism. -Support training priority and memory priority. \ No newline at end of file From c31e9c1d3a86eada4084dac52e802d72a2f2e069 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Sat, 15 Jul 2023 13:28:10 +0800 Subject: [PATCH 050/337] Update README.md --- README.md | 154 ++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 122 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 5375e96e..0bbbf10d 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,37 @@ -# Stop training and saving when condition is met: -## DL: +# Non-parallel training: + +## Save and restore: +```python +import Note.DL.kernel as k #import kernel +import tensorflow as tf #import platform +import nn as n #import neural network +mnist=tf.keras.datasets.mnist +(x_train,y_train),(x_test,y_test)=mnist.load_data() +x_train,x_test =x_train/255.0,x_test/255.0 +nn=n.nn() #create neural network object +kernel=k.kernel(nn) #start kernel +kernel.platform=tf #use platform +kernel.data(x_train,y_train) #input train data +kernel.train(32,5) #train neural network + #batch size:32 + #epoch:5 +kernel.save() #save neural network +``` +```python +import Note.DL.kernel as k #import kernel +import tensorflow as tf #import platform +mnist=tf.keras.datasets.mnist +(x_train,y_train),(x_test,y_test)=mnist.load_data() +x_train,x_test =x_train/255.0,x_test/255.0 +kernel=k.kernel() #start kernel +kernel.platform=tf #use platform +kernel.data(x_train,y_train) #input train data +kernel.restore('save.dat') #restore neural network +kernel.train(32,1) #train again +``` + +## Stop training and saving when condition is met: +### DL: ```python import Note.DL.kernel as k #import kernel import tensorflow as tf #import platform @@ -18,7 +50,7 @@ kernel.train(32,5) #train neural network #epoch:5 ``` -## RL: +### RL: ```python import Note.RL.kernel as k #import kernel import DQN as d @@ -34,8 +66,7 @@ kernel.reward #view reward kernel.visualize_reward() ``` - -# Training with test data +## Training with test data https://github.com/NoteDancing/Note-documentation/blob/Note-7.0/Note%207.0%20documentation/DL/neural%20network/tensorflow/nn_acc.py ```python import Note.DL.kernel as k #import kernel @@ -55,8 +86,7 @@ kernel.train(32,5,32) #train neural network kernel.save() #save neural network ``` - -# Parallel test: +## Parallel test: **You can download neural network example in this link,and then you can import neural network and train with kernel,link and example code are below.** https://github.com/NoteDancing/Note-documentation/blob/Note-7.0/Note%207.0%20documentation/DL/neural%20network/tensorflow/process/nn.py @@ -78,25 +108,12 @@ kernel.train(32,5,32) #train neural network #batch size:32 #epoch:5 ``` -```python -import tensorflow as tf -import nn_acc as n -import Note.DL.dl.test as t -mnist=tf.keras.datasets.mnist -(x_train,y_train),(x_test,y_test)=mnist.load_data() -x_train,x_test =x_train/255.0,x_test/255.0 -nn=n.nn() -nn.build() -test=t.parallel_test(nn,x_test,y_test,6,32) -test.segment_data() -for p in range(6): - Process(target=test.test).start() -loss=test.loss_acc() -``` -# Multiprocessing: +# Parallel training: + ## DL: + ### PO2: ```python import Note.DL.process.kernel as k #import kernel @@ -114,7 +131,7 @@ kernel.process=7 #7 processes to train kernel.data_segment_flag=True kernel.epoch=6 #epoch:6 kernel.batch=32 #batch:32 -kernel.PO=2 #use PO3 +kernel.PO=2 #use PO2 kernel.data(x_train,y_train) #input train data manager=Manager() #create manager object kernel.init(manager) #initialize shared data @@ -141,7 +158,7 @@ kernel.process=7 #7 processes to train kernel.data_segment_flag=True kernel.epoch=6 #epoch:6 kernel.batch=32 #batch:32 -kernel.PO=2 #use PO3 +kernel.PO=2 #use PO2 kernel.data(x_train,y_train) #input train data manager=Manager() #create manager object kernel.init(manager) #initialize shared data @@ -152,6 +169,7 @@ for p in range(7): kernel.update_nn_param() kernel.test(x_train,y_train,32) ``` + ### PO3： ```python import Note.DL.process.kernel as k #import kernel @@ -168,7 +186,7 @@ kernel.process=7 #7 processes to train kernel.data_segment_flag=True kernel.epoch=6 #epoch:6 kernel.batch=32 #batch:32 -kernel.PO=3 #use PO4 +kernel.PO=3 #use PO3 kernel.data(x_train,y_train) #input train data manager=Manager() #create manager object kernel.init(manager) #initialize shared data @@ -193,7 +211,7 @@ kernel.process=7 #7 processes to train kernel.data_segment_flag=True kernel.epoch=6 #epoch:6 kernel.batch=32 #batch:32 -kernel.PO=3 #use PO4 +kernel.PO=3 #use PO3 kernel.data(x_train,y_train) #input train data manager=Manager() #create manager object kernel.init(manager) #initialize shared data @@ -202,6 +220,55 @@ for p in range(7): kernel.update_nn_param() kernel.test(x_train,y_train,32) ``` + +### Save and restore: +```python +import Note.DL.process.kernel as k #import kernel +import tensorflow as tf +import nn as n #import neural network +from multiprocessing import Process,Lock,Manager +mnist=tf.keras.datasets.mnist +(x_train,y_train),(x_test,y_test)=mnist.load_data() +x_train,x_test =x_train/255.0,x_test/255.0 +nn=n.nn() #create neural network object +nn.build() +kernel=k.kernel(nn) #start kernel +kernel.process=7 #7 processes to train +kernel.data_segment_flag=True +kernel.epoch=6 #epoch:6 +kernel.batch=32 #batch:32 +kernel.PO=3 #use PO3 +kernel.data(x_train,y_train) #input train data +manager=Manager() #create manager object +kernel.init(manager) #initialize shared data +for p in range(7): + Process(target=kernel.train,args=(p,)).start() +kernel.update_nn_param() +kernel.test(x_train,y_train,32) +kernel.save() #save neural network +``` +```python +import Note.DL.process.kernel as k #import kernel +import tensorflow as tf +import nn as n #import neural network +from multiprocessing import Process,Lock,Manager +mnist=tf.keras.datasets.mnist +(x_train,y_train),(x_test,y_test)=mnist.load_data() +x_train,x_test =x_train/255.0,x_test/255.0 +kernel=k.kernel() #start kernel +kernel.process=7 #7 processes to train +kernel.data_segment_flag=True +kernel.epoch=1 #epoch:6 +kernel.batch=32 #batch:32 +kernel.PO=3 #use PO3 +kernel.data(x_train,y_train) #input train data +manager=Manager() #create manager object +kernel.init(manager) #initialize shared data +kernel.restore('save.dat') #restore neural network +for p in range(7): + Process(target=kernel.train,args=(p,)).start() +``` + ### Parallel test: ```python import Note.DL.process.kernel as k #import kernel @@ -227,6 +294,7 @@ lock=[Lock(),Lock()] for p in range(7): Process(target=kernel.train,args=(p,lock,None,32)).start() ``` + ### Stop training and saving when condition is met: ```python import Note.DL.process.kernel as k #import kernel @@ -246,7 +314,7 @@ kernel.process=7 #7 processes to train kernel.data_segment_flag=True kernel.epoch=6 #epoch:6 kernel.batch=32 #batch:32 -kernel.PO=2 #use PO3 +kernel.PO=2 #use PO2 kernel.data(x_train,y_train) #input train data manager=Manager() #create manager object kernel.init(manager) #initialize shared data @@ -256,14 +324,14 @@ for p in range(7): Process(target=kernel.train,args=(p,lock,g_lock)).start() ``` - ## RL： -### Pool Network: +**Pool Network:** + **You can download neural network example in this link,and then you can import neural network and train with kernel,link and example code are below.** https://github.com/NoteDancing/Note-documentation/blob/Note-7.0/Note%207.0%20documentation/RL/neural%20network/tensorflow/pool%20net/DQN.py -#### PO2: +### PO2: ```python import Note.RL.kernel as k #import kernel import DQN as d @@ -282,7 +350,7 @@ for p in range(5): Process(target=kernel.train,args=(p,100,lock,pool_lock,g_lock)).start() ``` -#### PO3: +### PO3: ```python import Note.RL.kernel as k #import kernel import DQN as d @@ -319,6 +387,28 @@ for p in range(5): ``` +# Parallel test: +**You can download neural network example in this link,and then you can import neural network and train with kernel,link and example code are below.** + +https://github.com/NoteDancing/Note-documentation/blob/Note-7.0/Note%207.0%20documentation/DL/neural%20network/tensorflow/process/nn.py + +```python +import tensorflow as tf +import nn_acc as n +import Note.DL.dl.test as t +mnist=tf.keras.datasets.mnist +(x_train,y_train),(x_test,y_test)=mnist.load_data() +x_train,x_test =x_train/255.0,x_test/255.0 +nn=n.nn() +nn.build() +test=t.parallel_test(nn,x_test,y_test,6,32) +test.segment_data() +for p in range(6): + Process(target=test.test).start() +loss=test.loss_acc() +``` + + # Online training: **You can download neural network example in this link,and then you can import neural network and train with kernel,link and example code are below.** From e7a91dfc1a78170caa2b02bda7674c67a7a10b9c Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Sat, 15 Jul 2023 20:11:20 +0800 Subject: [PATCH 051/337] Update README.md --- README.md | 679 +++++++++++++++++++++++++++--------------------------- 1 file changed, 335 insertions(+), 344 deletions(-) diff --git a/README.md b/README.md index 0bbbf10d..64e54edc 100644 --- a/README.md +++ b/README.md @@ -2,88 +2,79 @@ ## Save and restore: ```python -import Note.DL.kernel as k #import kernel -import tensorflow as tf #import platform -import nn as n #import neural network -mnist=tf.keras.datasets.mnist -(x_train,y_train),(x_test,y_test)=mnist.load_data() -x_train,x_test =x_train/255.0,x_test/255.0 -nn=n.nn() #create neural network object -kernel=k.kernel(nn) #start kernel -kernel.platform=tf #use platform -kernel.data(x_train,y_train) #input train data -kernel.train(32,5) #train neural network - #batch size:32 - #epoch:5 -kernel.save() #save neural network +import Note.DL.kernel as k #import kernel module +import tensorflow as tf #import tensorflow library +import nn as n #import neural network module +mnist=tf.keras.datasets.mnist #load mnist dataset +(x_train,y_train),(x_test,y_test)=mnist.load_data() #split data into train and test sets +x_train,x_test =x_train/255.0,x_test/255.0 #normalize data +nn=n.nn() #create neural network object +kernel=k.kernel(nn) #create kernel object with the network +kernel.platform=tf #set the platform to tensorflow +kernel.data(x_train,y_train) #input train data to the kernel +kernel.train(32,5) #train the network with batch size 32 and epoch 5 +kernel.save() #save the neural network to a file ``` ```python -import Note.DL.kernel as k #import kernel -import tensorflow as tf #import platform -mnist=tf.keras.datasets.mnist -(x_train,y_train),(x_test,y_test)=mnist.load_data() -x_train,x_test =x_train/255.0,x_test/255.0 -kernel=k.kernel() #start kernel -kernel.platform=tf #use platform -kernel.data(x_train,y_train) #input train data -kernel.restore('save.dat') #restore neural network -kernel.train(32,1) #train again +import Note.DL.kernel as k #import kernel module +import tensorflow as tf #import tensorflow library +mnist=tf.keras.datasets.mnist #load mnist dataset +(x_train,y_train),(x_test,y_test)=mnist.load_data() #split data into train and test sets +x_train,x_test =x_train/255.0,x_test/255.0 #normalize data +kernel=k.kernel() #create kernel object without a network +kernel.platform=tf #set the platform to tensorflow +kernel.data(x_train,y_train) #input train data to the kernel +kernel.restore('save.dat') #restore the network from a file +kernel.train(32,1) #train the network again with batch size 32 and epoch 1 ``` ## Stop training and saving when condition is met: ### DL: ```python -import Note.DL.kernel as k #import kernel -import tensorflow as tf #import platform -import nn as n #import neural network -mnist=tf.keras.datasets.mnist -(x_train,y_train),(x_test,y_test)=mnist.load_data() -x_train,x_test =x_train/255.0,x_test/255.0 -nn=n.nn() #create neural network object -kernel=k.kernel(nn) #start kernel -kernel.platform=tf #use platform -kernel.stop=True -kernel.end_loss=0.7 -kernel.data(x_train,y_train) #input train data -kernel.train(32,5) #train neural network - #batch size:32 - #epoch:5 +import Note.DL.kernel as k #import kernel module +import tensorflow as tf #import tensorflow library +import nn as n #import neural network module +mnist=tf.keras.datasets.mnist #load mnist dataset +(x_train,y_train),(x_test,y_test)=mnist.load_data() #split data into train and test sets +x_train,x_test =x_train/255.0,x_test/255.0 #normalize data +nn=n.nn() #create neural network object +kernel=k.kernel(nn) #create kernel object with the network +kernel.platform=tf #set the platform to tensorflow +kernel.stop=True #set the flag to stop training when a condition is met +kernel.end_loss=0.7 #set the condition to stop training when the loss is less than 0.7 +kernel.data(x_train,y_train) #input train data to the kernel +kernel.train(32,5) #train the network with batch size 32 and epoch 5 ``` ### RL: ```python -import Note.RL.kernel as k #import kernel -import DQN as d -dqn=d.DQN(4,128,2) #create neural network object -kernel=k.kernel(dqn,5) #start kernel -kernel.stop=True -kernel.action_count=2 -kernel.set_up(epsilon=0.01,pool_size=10000,batch=64,update_step=10,trial_count=10,criterion=200) -kernel.train(500) -kernel.loss_list or kernel.loss #view training loss +import Note.RL.kernel as k #import kernel module +import DQN as d #import deep Q-network module +dqn=d.DQN(4,128,2) #create neural network object with 4 inputs, 128 hidden units and 2 outputs +kernel=k.kernel(dqn,5) #create kernel object with the network and 5 threads to train +kernel.stop=True #set the flag to stop training when a condition is met +kernel.action_count=2 #set the number of actions to 2 +kernel.set_up(epsilon=0.01,pool_size=10000,batch=64,update_step=10,trial_count=10,criterion=200) #set up the hyperparameters for training and the condition to stop training when the average reward of 10 trials is greater than 200 +kernel.train(500) #train the network for 500 episodes kernel.visualize_train() -kernel.reward #view reward kernel.visualize_reward() ``` ## Training with test data https://github.com/NoteDancing/Note-documentation/blob/Note-7.0/Note%207.0%20documentation/DL/neural%20network/tensorflow/nn_acc.py ```python -import Note.DL.kernel as k #import kernel -import tensorflow as tf #import platform -import nn_acc as n #import neural network -mnist=tf.keras.datasets.mnist -(x_train,y_train),(x_test,y_test)=mnist.load_data() -x_train,x_test =x_train/255.0,x_test/255.0 -nn=n.nn() #create neural network object -kernel=k.kernel(nn) #start kernel -kernel.platform=tf #use platform -kernel.data(x_train,y_train,x_test,y_test) #input train data -kernel.train(32,5,32) #train neural network - #batch size:32 - #test batch size:32 - #epoch:5 -kernel.save() #save neural network +import Note.DL.kernel as k #import kernel module +import tensorflow as tf #import tensorflow library +import nn_acc as n #import neural network module with accuracy function +mnist=tf.keras.datasets.mnist #load mnist dataset +(x_train,y_train),(x_test,y_test)=mnist.load_data() #split data into train and test sets +x_train,x_test =x_train/255.0,x_test/255.0 #normalize data +nn=n.nn() #create neural network object +kernel=k.kernel(nn) #create kernel object with the network +kernel.platform=tf #set the platform to tensorflow +kernel.data(x_train,y_train,x_test,y_test) #input train and test data and labels to the kernel +kernel.train(32,5,32) #train the network with batch size 32, epoch 5 and test batch size 32 +kernel.test(x_test,y_test,32)#test the network performance on the test set with batch size 32 ``` ## Parallel test: @@ -92,21 +83,19 @@ kernel.save() #save neural network https://github.com/NoteDancing/Note-documentation/blob/Note-7.0/Note%207.0%20documentation/DL/neural%20network/tensorflow/process/nn.py ```python -import Note.DL.kernel as k #import kernel -import tensorflow as tf #import platform -import nn as n #import neural network -mnist=tf.keras.datasets.mnist -(x_train,y_train),(x_test,y_test)=mnist.load_data() -x_train,x_test =x_train/255.0,x_test/255.0 -nn=n.nn() #create neural network object -nn.build() -kernel=k.kernel(nn) #start kernel -kernel.platform=tf #use platform -kernel.process_t=3 -kernel.data(x_train,y_train,x_test,y_test) #input train data -kernel.train(32,5,32) #train neural network - #batch size:32 - #epoch:5 +import Note.DL.kernel as k #import kernel module +import tensorflow as tf #import tensorflow library +import nn as n #import neural network module +mnist=tf.keras.datasets.mnist #load mnist dataset +(x_train,y_train),(x_test,y_test)=mnist.load_data() #split data into train and test sets +x_train,x_test =x_train/255.0,x_test/255.0 #normalize data +nn=n.nn() #create neural network object +nn.build() #build the network structure +kernel=k.kernel(nn) #create kernel object with the network +kernel.platform=tf #set the platform to tensorflow +kernel.process_t=3 #set the number of processes to test +kernel.data(x_train,y_train,x_test,y_test) #input train and test data to the kernel +kernel.train(32,5,32) #train the network with batch size 32, epoch 5 and test batch size 32 ``` @@ -116,212 +105,212 @@ kernel.train(32,5,32) #train neural network ### PO2: ```python -import Note.DL.process.kernel as k #import kernel -import tensorflow as tf -import nn as n #import neural network -from multiprocessing import Process,Lock,Manager -mnist=tf.keras.datasets.mnist -(x_train,y_train),(x_test,y_test)=mnist.load_data() -x_train,x_test =x_train/255.0,x_test/255.0 -x_train=x_train.reshape([60000,784]) -nn=n.nn() #create neural network object -nn.build() -kernel=k.kernel(nn) #start kernel -kernel.process=7 #7 processes to train -kernel.data_segment_flag=True -kernel.epoch=6 #epoch:6 -kernel.batch=32 #batch:32 -kernel.PO=2 #use PO2 -kernel.data(x_train,y_train) #input train data -manager=Manager() #create manager object -kernel.init(manager) #initialize shared data -lock=[Lock(),Lock()] -g_lock=Lock() -for p in range(7): - Process(target=kernel.train,args=(p,lock,g_lock)).start() -kernel.update_nn_param() -kernel.test(x_train,y_train,32) +import Note.DL.process.kernel as k #import kernel module +import tensorflow as tf #import tensorflow library +import nn as n #import neural network module +from multiprocessing import Process,Lock,Manager #import multiprocessing tools +mnist=tf.keras.datasets.mnist #load mnist dataset +(x_train,y_train),(x_test,y_test)=mnist.load_data() #split data into train and test sets +x_train,x_test =x_train/255.0,x_test/255.0 #normalize data +x_train=x_train.reshape([60000,784]) #reshape data to fit the network input +nn=n.nn() #create neural network object +nn.build() #build the network structure +kernel=k.kernel(nn) #create kernel object with the network +kernel.process=7 #set the number of processes to train +kernel.data_segment_flag=True #set the flag to segment data +kernel.epoch=6 #set the number of epochs to train +kernel.batch=32 #set the batch size +kernel.PO=2 #use PO2 algorithm for parallel optimization +kernel.data(x_train,y_train) #input train data to the kernel +manager=Manager() #create manager object to share data among processes +kernel.init(manager) #initialize shared data with the manager +lock=[Lock(),Lock()] #create two locks for synchronization +g_lock=Lock() #create a global lock for gradient computing +for p in range(7): #loop over the processes + Process(target=kernel.train,args=(p,lock,g_lock)).start() #start each process with the train function and pass the process id and locks as arguments +kernel.update_nn_param() #update the network parameters after training +kernel.test(x_train,y_train,32) #test the network performance on the train set with batch size 32 ``` ```python -import Note.DL.process.kernel as k #import kernel -import tensorflow as tf -import nn as n #import neural network -from multiprocessing import Process,Lock,Manager -mnist=tf.keras.datasets.mnist -(x_train,y_train),(x_test,y_test)=mnist.load_data() -x_train,x_test =x_train/255.0,x_test/255.0 -x_train=x_train.reshape([60000,784]) -nn=n.nn() #create neural network object -nn.build() -kernel=k.kernel(nn) #start kernel -kernel.process=7 #7 processes to train -kernel.data_segment_flag=True -kernel.epoch=6 #epoch:6 -kernel.batch=32 #batch:32 -kernel.PO=2 #use PO2 -kernel.data(x_train,y_train) #input train data -manager=Manager() #create manager object -kernel.init(manager) #initialize shared data -lock=[Lock(),Lock()] -g_lock=Lock() -for p in range(7): - Process(target=kernel.train,args=(p,lock,g_lock)).start() -kernel.update_nn_param() -kernel.test(x_train,y_train,32) +import Note.DL.process.kernel as k #import kernel module +import tensorflow as tf #import tensorflow library +import nn as n #import neural network module +from multiprocessing import Process,Lock,Manager #import multiprocessing tools +mnist=tf.keras.datasets.mnist #load mnist dataset +(x_train,y_train),(x_test,y_test)=mnist.load_data() #split data into train and test sets +x_train,x_test =x_train/255.0,x_test/255.0 #normalize data +x_train=x_train.reshape([60000,784]) #reshape data to fit the network input +nn=n.nn() #create neural network object +nn.build() #build the network structure +kernel=k.kernel(nn) #create kernel object with the network +kernel.process=7 #set the number of processes to train +kernel.data_segment_flag=True #set the flag to segment data +kernel.epoch=6 #set the number of epochs to train +kernel.batch=32 #set the batch size +kernel.PO=2 #use PO2 algorithm for parallel optimization +kernel.data(x_train,y_train) #input train data to the kernel +manager=Manager() #create manager object to share data among processes +kernel.init(manager) #initialize shared data with the manager +lock=[Lock(),Lock()] #create two locks for synchronization +g_lock=[Lock(),Lock()] #create a list of global locks for gradient computing +for p in range(7): #loop over the processes + Process(target=kernel.train,args=(p,lock,g_lock)).start() #start each process with the train function and pass the process id, the locks and the global locks as arguments +kernel.update_nn_param() #update the network parameters after training +kernel.test(x_train,y_train,32) #test the network performance on the train set with batch size 32 ``` ### PO3： ```python -import Note.DL.process.kernel as k #import kernel -import tensorflow as tf -import nn as n #import neural network -from multiprocessing import Process,Lock,Manager -mnist=tf.keras.datasets.mnist -(x_train,y_train),(x_test,y_test)=mnist.load_data() -x_train,x_test =x_train/255.0,x_test/255.0 -nn=n.nn() #create neural network object -nn.build() -kernel=k.kernel(nn) #start kernel -kernel.process=7 #7 processes to train -kernel.data_segment_flag=True -kernel.epoch=6 #epoch:6 -kernel.batch=32 #batch:32 -kernel.PO=3 #use PO3 -kernel.data(x_train,y_train) #input train data -manager=Manager() #create manager object -kernel.init(manager) #initialize shared data -lock=Lock() -for p in range(7): - Process(target=kernel.train,args=(p,lock)).start() -kernel.update_nn_param() -kernel.test(x_train,y_train,32) +import Note.DL.process.kernel as k #import kernel module +import tensorflow as tf #import tensorflow library +import nn as n #import neural network module +from multiprocessing import Process,Lock,Manager #import multiprocessing tools +mnist=tf.keras.datasets.mnist #load mnist dataset +(x_train,y_train),(x_test,y_test)=mnist.load_data() #split data into train and test sets +x_train,x_test =x_train/255.0,x_test/255.0 #normalize data +nn=n.nn() #create neural network object +nn.build() #build the network structure +kernel=k.kernel(nn) #create kernel object with the network +kernel.process=7 #set the number of processes to train +kernel.data_segment_flag=True #set the flag to segment data +kernel.epoch=6 #set the number of epochs to train +kernel.batch=32 #set the batch size +kernel.PO=3 #use PO3 algorithm for parallel optimization +kernel.data(x_train,y_train) #input train data to the kernel +manager=Manager() #create manager object to share data among processes +kernel.init(manager) #initialize shared data with the manager +lock=Lock() #create a lock for synchronization +for p in range(7): #loop over the processes + Process(target=kernel.train,args=(p,lock)).start() #start each process with the train function and pass the process id and the lock as arguments +kernel.update_nn_param() #update the network parameters after training +kernel.test(x_train,y_train,32) #test the network performance on the train set with batch size 32 ``` ```python -import Note.DL.process.kernel as k #import kernel -import tensorflow as tf -import nn as n #import neural network -from multiprocessing import Process,Lock,Manager -mnist=tf.keras.datasets.mnist -(x_train,y_train),(x_test,y_test)=mnist.load_data() -x_train,x_test =x_train/255.0,x_test/255.0 -nn=n.nn() #create neural network object -nn.build() -kernel=k.kernel(nn) #start kernel -kernel.process=7 #7 processes to train -kernel.data_segment_flag=True -kernel.epoch=6 #epoch:6 -kernel.batch=32 #batch:32 -kernel.PO=3 #use PO3 -kernel.data(x_train,y_train) #input train data -manager=Manager() #create manager object -kernel.init(manager) #initialize shared data -for p in range(7): - Process(target=kernel.train,args=(p,)).start() -kernel.update_nn_param() -kernel.test(x_train,y_train,32) +import Note.DL.process.kernel as k #import kernel module +import tensorflow as tf #import tensorflow library +import nn as n #import neural network module +from multiprocessing import Process,Lock,Manager #import multiprocessing tools +mnist=tf.keras.datasets.mnist #load mnist dataset +(x_train,y_train),(x_test,y_test)=mnist.load_data() #split data into train and test sets +x_train,x_test =x_train/255.0,x_test/255.0 #normalize data +nn=n.nn() #create neural network object +nn.build() #build the network structure +kernel=k.kernel(nn) #create kernel object with the network +kernel.process=7 #set the number of processes to train +kernel.data_segment_flag=True #set the flag to segment data +kernel.epoch=6 #set the number of epochs to train +kernel.batch=32 #set the batch size +kernel.PO=3 #use PO3 algorithm for parallel optimization +kernel.data(x_train,y_train) #input train data to the kernel +manager=Manager() #create manager object to share data among processes +kernel.init(manager) #initialize shared data with the manager +for p in range(7): #loop over the processes + Process(target=kernel.train,args=(p,)).start() #start each process with the train function and pass the process id as argument +kernel.update_nn_param() #update the network parameters after training +kernel.test(x_train,y_train,32) #test the network performance on the train set with batch size 32 + ``` ### Save and restore: ```python -import Note.DL.process.kernel as k #import kernel -import tensorflow as tf -import nn as n #import neural network -from multiprocessing import Process,Lock,Manager -mnist=tf.keras.datasets.mnist -(x_train,y_train),(x_test,y_test)=mnist.load_data() -x_train,x_test =x_train/255.0,x_test/255.0 -nn=n.nn() #create neural network object -nn.build() -kernel=k.kernel(nn) #start kernel -kernel.process=7 #7 processes to train -kernel.data_segment_flag=True -kernel.epoch=6 #epoch:6 -kernel.batch=32 #batch:32 -kernel.PO=3 #use PO3 -kernel.data(x_train,y_train) #input train data -manager=Manager() #create manager object -kernel.init(manager) #initialize shared data -for p in range(7): - Process(target=kernel.train,args=(p,)).start() -kernel.update_nn_param() -kernel.test(x_train,y_train,32) -kernel.save() #save neural network +import Note.DL.process.kernel as k #import kernel module +import tensorflow as tf #import tensorflow library +import nn as n #import neural network module +from multiprocessing import Process,Lock,Manager #import multiprocessing tools +mnist=tf.keras.datasets.mnist #load mnist dataset +(x_train,y_train),(x_test,y_test)=mnist.load_data() #split data into train and test sets +x_train,x_test =x_train/255.0,x_test/255.0 #normalize data +nn=n.nn() #create neural network object +nn.build() #build the network structure +kernel=k.kernel(nn) #create kernel object with the network +kernel.process=7 #set the number of processes to train +kernel.data_segment_flag=True #set the flag to segment data +kernel.epoch=6 #set the number of epochs to train +kernel.batch=32 #set the batch size +kernel.PO=3 #use PO3 algorithm for parallel optimization +kernel.data(x_train,y_train) #input train data to the kernel +manager=Manager() #create manager object to share data among processes +kernel.init(manager) #initialize shared data with the manager +for p in range(7): #loop over the processes + Process(target=kernel.train,args=(p,)).start() #start each process with the train function and pass the process id as argument +kernel.update_nn_param() #update the network parameters after training +kernel.test(x_train,y_train,32) #test the network performance on the train set with batch size 32 +kernel.save() #save the neural network to a file ``` ```python -import Note.DL.process.kernel as k #import kernel -import tensorflow as tf -import nn as n #import neural network -from multiprocessing import Process,Lock,Manager -mnist=tf.keras.datasets.mnist -(x_train,y_train),(x_test,y_test)=mnist.load_data() -x_train,x_test =x_train/255.0,x_test/255.0 -kernel=k.kernel() #start kernel -kernel.process=7 #7 processes to train -kernel.data_segment_flag=True -kernel.epoch=1 #epoch:6 -kernel.batch=32 #batch:32 -kernel.PO=3 #use PO3 -kernel.data(x_train,y_train) #input train data -manager=Manager() #create manager object -kernel.init(manager) #initialize shared data -kernel.restore('save.dat') #restore neural network -for p in range(7): - Process(target=kernel.train,args=(p,)).start() +import Note.DL.process.kernel as k #import kernel module +import tensorflow as tf #import tensorflow library +import nn as n #import neural network module +from multiprocessing import Process,Lock,Manager #import multiprocessing tools +mnist=tf.keras.datasets.mnist #load mnist dataset +(x_train,y_train),(x_test,y_test)=mnist.load_data() #split data into train and test sets +x_train,x_test =x_train/255.0,x_test/255.0 #normalize data +kernel=k.kernel() #create kernel object without a network +kernel.process=7 #set the number of processes to train +kernel.data_segment_flag=True #set the flag to segment data +kernel.epoch=1 #set the number of epochs to train +kernel.batch=32 #set the batch size +kernel.PO=3 #use PO3 algorithm for parallel optimization +kernel.data(x_train,y_train) #input train data to the kernel +manager=Manager() #create manager object to share data among processes +kernel.init(manager) #initialize shared data with the manager +kernel.restore('save.dat') #restore the neural network from a file +for p in range(7): #loop over the processes + Process(target=kernel.train,args=(p,)).start() #start each process with the train function and pass the process id as argument ``` ### Parallel test: ```python -import Note.DL.process.kernel as k #import kernel -import tensorflow as tf -import nn as n #import neural network -from multiprocessing import Process,Lock,Manager -mnist=tf.keras.datasets.mnist -(x_train,y_train),(x_test,y_test)=mnist.load_data() -x_train,x_test =x_train/255.0,x_test/255.0 -nn=n.nn() #create neural network object -nn.build() -kernel=k.kernel(nn) #start kernel -kernel.process=7 #7 processes to train -kernel.process_t=3 -kernel.data_segment_flag=True -kernel.epoch=6 #epoch:6 -kernel.batch=32 #batch:32 -kernel.PO=1 #use PO1 -kernel.data(x_train,y_train,x_test,y_test) #input train data -manager=Manager() #create manager object -kernel.init(manager) #initialize shared data -lock=[Lock(),Lock()] -for p in range(7): - Process(target=kernel.train,args=(p,lock,None,32)).start() +import Note.DL.process.kernel as k #import kernel module +import tensorflow as tf #import tensorflow library +import nn as n #import neural network module +from multiprocessing import Process,Lock,Manager #import multiprocessing tools +mnist=tf.keras.datasets.mnist #load mnist dataset +(x_train,y_train),(x_test,y_test)=mnist.load_data() #split data into train and test sets +x_train,x_test =x_train/255.0,x_test/255.0 #normalize data +nn=n.nn() #create neural network object +nn.build() #build the network structure +kernel=k.kernel(nn) #create kernel object with the network +kernel.process=7 #set the number of processes to train +kernel.process_t=3 #set the number of processes to test +kernel.data_segment_flag=True #set the flag to segment data +kernel.epoch=6 #set the number of epochs to train +kernel.batch=32 #set the batch size +kernel.PO=3 #use PO3 algorithm for parallel optimization +kernel.data(x_train,y_train,x_test,y_test) #input train and test data to the kernel +manager=Manager() #create manager object to share data among processes +kernel.init(manager) #initialize shared data with the manager +for p in range(7): #loop over the processes + Process(target=kernel.train,args=(p,None,None,32)).start() #start each process with the train function and pass the process id, the locks, the test flag and the test batch size as arguments ``` ### Stop training and saving when condition is met: ```python -import Note.DL.process.kernel as k #import kernel -import tensorflow as tf -import nn as n #import neural network -from multiprocessing import Process,Lock,Manager -mnist=tf.keras.datasets.mnist -(x_train,y_train),(x_test,y_test)=mnist.load_data() -x_train,x_test =x_train/255.0,x_test/255.0 -x_train=x_train.reshape([60000,784]) -nn=n.nn() #create neural network object -nn.build() -kernel=k.kernel(nn) #start kernel -kernel.stop=True -kernel.end_loss=0.7 #stop condition -kernel.process=7 #7 processes to train -kernel.data_segment_flag=True -kernel.epoch=6 #epoch:6 -kernel.batch=32 #batch:32 -kernel.PO=2 #use PO2 -kernel.data(x_train,y_train) #input train data -manager=Manager() #create manager object -kernel.init(manager) #initialize shared data -lock=[Lock(),Lock()] -g_lock=Lock() -for p in range(7): - Process(target=kernel.train,args=(p,lock,g_lock)).start() +import Note.DL.process.kernel as k #import kernel module +import tensorflow as tf #import tensorflow library +import nn as n #import neural network module +from multiprocessing import Process,Lock,Manager #import multiprocessing tools +mnist=tf.keras.datasets.mnist #load mnist dataset +(x_train,y_train),(x_test,y_test)=mnist.load_data() #split data into train and test sets +x_train,x_test =x_train/255.0,x_test/255.0 #normalize data +x_train=x_train.reshape([60000,784]) #reshape data to fit the network input +nn=n.nn() #create neural network object +nn.build() #build the network structure +kernel=k.kernel(nn) #create kernel object with the network +kernel.stop=True #set the flag to stop training when a condition is met +kernel.end_loss=0.7 #set the condition to stop training when the loss is less than 0.7 +kernel.process=7 #set the number of processes to train +kernel.data_segment_flag=True #set the flag to segment data +kernel.epoch=6 #set the number of epochs to train +kernel.batch=32 #set the batch size +kernel.PO=2 #use PO2 algorithm for parallel optimization +kernel.data(x_train,y_train) #input train data to the kernel +manager=Manager() #create manager object to share data among processes +kernel.init(manager) #initialize shared data with the manager +lock=[Lock(),Lock()] #create two locks for synchronization +g_lock=Lock() #create a global lock for gradient update +for p in range(7): #loop over the processes + Process(target=kernel.train,args=(p,lock,g_lock)).start() #start each process with the train function and pass the process id and locks as arguments ``` ## RL： @@ -333,57 +322,57 @@ https://github.com/NoteDancing/Note-documentation/blob/Note-7.0/Note%207.0%20doc ### PO2: ```python -import Note.RL.kernel as k #import kernel -import DQN as d -from multiprocessing import Process,Lock,Manager -dqn=d.DQN(4,128,2) #create neural network object -kernel=k.kernel(dqn,5) #start kernel,use 5 thread to train -manager=Manager() #create manager object -kernel.init(manager) #initialize shared data -kernel.action_count=2 -kernel.set_up(epsilon=0.01,pool_size=10000,batch=64,update_step=10) -kernel.PO=2 #use PO2 -pool_lock=[Lock(),Lock(),Lock(),Lock(),Lock()] -lock=[Lock(),Lock(),Lock()] -g_lock=Lock() -for p in range(5): - Process(target=kernel.train,args=(p,100,lock,pool_lock,g_lock)).start() +import Note.RL.kernel as k #import kernel module +import DQN as d #import deep Q-network module +from multiprocessing import Process,Lock,Manager #import multiprocessing tools +dqn=d.DQN(4,128,2) #create neural network object with 4 inputs, 128 hidden units and 2 outputs +kernel=k.kernel(dqn,5) #create kernel object with the network and 5 processes to train +manager=Manager() #create manager object to share data among processes +kernel.init(manager) #initialize shared data with the manager +kernel.action_count=2 #set the number of actions to 2 +kernel.set_up(epsilon=0.01,pool_size=10000,batch=64,update_step=10) #set up the hyperparameters for training +kernel.PO=2 #use PO2 algorithm for parallel optimization +pool_lock=[Lock(),Lock(),Lock(),Lock(),Lock()] #create a list of locks for each process's replay pool +lock=[Lock(),Lock(),Lock()] #create a list of locks for synchronization +g_lock=Lock() #create a global lock for gradient computing +for p in range(5): #loop over the processes + Process(target=kernel.train,args=(p,100,lock,pool_lock,g_lock)).start() #start each process with the train function and pass the process id, the number of episodes, the locks, the pool locks and the global lock as arguments ``` ### PO3: ```python -import Note.RL.kernel as k #import kernel -import DQN as d -from multiprocessing import Process,Lock,Manager -dqn=d.DQN(4,128,2) #create neural network object -kernel=k.kernel(dqn,5) #start kernel,use 5 thread to train -manager=Manager() #create manager object -kernel.init(manager) #initialize shared data -kernel.action_count=2 -kernel.set_up(epsilon=0.01,pool_size=10000,batch=64,update_step=10) -kernel.PO=3 #use PO3 -pool_lock=[Lock(),Lock(),Lock(),Lock(),Lock()] -lock=[Lock(),Lock()] -g_lock=Lock() -for p in range(5): - Process(target=kernel.train,args=(p,100,lock,pool_lock)).start() +import Note.RL.kernel as k #import kernel module +import DQN as d #import deep Q-network module +from multiprocessing import Process,Lock,Manager #import multiprocessing tools +dqn=d.DQN(4,128,2) #create neural network object with 4 inputs, 128 hidden units and 2 outputs +kernel=k.kernel(dqn,5) #create kernel object with the network and 5 processes to train +manager=Manager() #create manager object to share data among processes +kernel.init(manager) #initialize shared data with the manager +kernel.action_count=2 #set the number of actions to 2 +kernel.set_up(epsilon=0.01,pool_size=10000,batch=64,update_step=10) #set up the hyperparameters for training +kernel.PO=3 #use PO3 algorithm for parallel optimization +pool_lock=[Lock(),Lock(),Lock(),Lock(),Lock()] #create a list of locks for each process's replay pool +lock=[Lock(),Lock()] #create two locks for synchronization +g_lock=Lock() #create a global lock for gradient update +for p in range(5): #loop over the processes + Process(target=kernel.train,args=(p,100,lock,pool_lock)).start() #start each process with the train function and pass the process id, the number of episodes, the locks and the pool locks as arguments ``` ```python -import Note.RL.kernel as k #import kernel -import DQN as d -from multiprocessing import Process,Lock,Manager -dqn=d.DQN(4,128,2) #create neural network object -kernel=k.kernel(dqn,5) #start kernel,use 5 thread to train -manager=Manager() #create manager object -kernel.init(manager) #initialize shared data -kernel.action_count=2 -kernel.set_up(epsilon=0.01,pool_size=10000,batch=64,update_step=10) -kernel.PO=3 #use PO3 -pool_lock=[Lock(),Lock(),Lock(),Lock(),Lock()] -lock=[Lock(),Lock(),Lock()] -g_lock=Lock() -for p in range(5): - Process(target=kernel.train,args=(p,100,lock,pool_lock)).start() +import Note.RL.kernel as k #import kernel module +import DQN as d #import deep Q-network module +from multiprocessing import Process,Lock,Manager #import multiprocessing tools +dqn=d.DQN(4,128,2) #create neural network object with 4 inputs, 128 hidden units and 2 outputs +kernel=k.kernel(dqn,5) #create kernel object with the network and 5 processes to train +manager=Manager() #create manager object to share data among processes +kernel.init(manager) #initialize shared data with the manager +kernel.action_count=2 #set the number of actions to 2 +kernel.set_up(epsilon=0.01,pool_size=10000,batch=64,update_step=10) #set up the hyperparameters for training +kernel.PO=3 #use PO3 algorithm for parallel optimization +pool_lock=[Lock(),Lock(),Lock(),Lock(),Lock()] #create a list of locks for each process's replay pool +lock=[Lock(),Lock(),Lock()] #create a list of locks for synchronization +g_lock=Lock() #create a global lock for gradient update +for p in range(5): #loop over the processes + Process(target=kernel.train,args=(p,100,lock,pool_lock)).start() #start each process with the train function and pass the process id, the number of episodes, the locks and the pool locks as arguments ``` @@ -393,19 +382,20 @@ for p in range(5): https://github.com/NoteDancing/Note-documentation/blob/Note-7.0/Note%207.0%20documentation/DL/neural%20network/tensorflow/process/nn.py ```python -import tensorflow as tf -import nn_acc as n -import Note.DL.dl.test as t -mnist=tf.keras.datasets.mnist -(x_train,y_train),(x_test,y_test)=mnist.load_data() -x_train,x_test =x_train/255.0,x_test/255.0 -nn=n.nn() -nn.build() -test=t.parallel_test(nn,x_test,y_test,6,32) -test.segment_data() -for p in range(6): - Process(target=test.test).start() -loss=test.loss_acc() +import tensorflow as tf #import tensorflow library +from multiprocessing import Process #import multiprocessing tools +import nn_acc as n #import neural network module with accuracy function +import Note.DL.dl.test as t #import parallel test module +mnist=tf.keras.datasets.mnist #load mnist dataset +(x_train,y_train),(x_test,y_test)=mnist.load_data() #split data into train and test sets +x_train,x_test =x_train/255.0,x_test/255.0 #normalize data +nn=n.nn() #create neural network object +nn.build() #build the network structure +test=t.parallel_test(nn,x_test,y_test,6,32) #create parallel test object with the network, the test data, the number of processes and the batch size +test.segment_data() #segment data +for p in range(6): #loop over the processes + Process(target=test.test).start() #start each process with the test function +loss=test.loss_acc() #calculate the loss and accuracy of the test ``` @@ -416,16 +406,17 @@ https://github.com/NoteDancing/Note-documentation/blob/Note-7.0/Note%207.0%20doc **example:** ```python -import Note.DL.kernel as k #import kernel #import platform -import nn_ol as n #import neural network -mnist=tf.keras.datasets.mnist -(x_train,y_train),(x_test,y_test)=mnist.load_data() -x_train,x_test =x_train/255.0,x_test/255.0 -nn=n.nn(x_train,y_train) #create neural network object -kernel=k.kernel(nn) #start kernel -kernel.platform=tf #use platform -kernel.data(x_train,y_train) #input train data -kernel.train_ol() #train neural network +import Note.DL.kernel as k #import kernel module +import tensorflow as tf #import tensorflow library +import nn_ol as n #import neural network module with online learning +mnist=tf.keras.datasets.mnist #load mnist dataset +(x_train,y_train),(x_test,y_test)=mnist.load_data() #split data into train and test sets +x_train,x_test =x_train/255.0,x_test/255.0 #normalize data +nn=n.nn(x_train,y_train) #create neural network object with train data and labels +kernel=k.kernel(nn) #create kernel object with the network +kernel.platform=tf #set the platform to tensorflow +kernel.data(x_train,y_train) #input train data and labels to the kernel +kernel.train_online() #train the network online ``` @@ -433,24 +424,24 @@ kernel.train_ol() #train neural network ## DL: You can test it before using the kernel training neural network. ```python -from Note.DL.dl.check_nn import check -import tensorflow as tf #import platform -import nn as n #import neural network -mnist=tf.keras.datasets.mnist -(x_train,y_train),(x_test,y_test)=mnist.load_data() -x_train,x_test =x_train/255.0,x_test/255.0 -nn=n.nn() -check(cnn,tf,x_train[:32],y_train[:32]) +from Note.DL.dl.check_nn import check #import check function from check_nn module +import tensorflow as tf #import tensorflow library +import nn as n #import neural network module +mnist=tf.keras.datasets.mnist #load mnist dataset +(x_train,y_train),(x_test,y_test)=mnist.load_data() #split data into train and test sets +x_train,x_test =x_train/255.0,x_test/255.0 #normalize data +nn=n.nn() #create neural network object +check(nn,tf,x_train[:32],y_train[:32]) #check the network with tensorflow platform and a sample of 32 data points and labels ``` ## RL: You can test it before using the kernel training neural network. ```python -from Note.RL.rl.check_nn import check -import tensorflow as tf #import platform -import DQN as d -dqn=d.DQN(4,128,2) #create neural network object -checkdqn,tf,2) +from Note.RL.rl.check_nn import check #import check function from check_nn module +import tensorflow as tf #import tensorflow library +import DQN as d #import deep Q-network module +dqn=d.DQN(4,128,2) #create neural network object with 4 inputs, 128 hidden units and 2 outputs +check(dqn,tf,2) #check the network with tensorflow platform and 2 actions ``` From b994e043e2cb2419d509a5e3a8166be71d6526bb Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Sat, 15 Jul 2023 23:02:00 +0800 Subject: [PATCH 052/337] Update README.md --- README.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 64e54edc..7e1f3085 100644 --- a/README.md +++ b/README.md @@ -117,7 +117,7 @@ nn=n.nn() #create neural network object nn.build() #build the network structure kernel=k.kernel(nn) #create kernel object with the network kernel.process=7 #set the number of processes to train -kernel.data_segment_flag=True #set the flag to segment data +kernel.data_segment_flag=True #set the flag to segment data for each process kernel.epoch=6 #set the number of epochs to train kernel.batch=32 #set the batch size kernel.PO=2 #use PO2 algorithm for parallel optimization @@ -144,7 +144,7 @@ nn=n.nn() #create neural network object nn.build() #build the network structure kernel=k.kernel(nn) #create kernel object with the network kernel.process=7 #set the number of processes to train -kernel.data_segment_flag=True #set the flag to segment data +kernel.data_segment_flag=True #set the flag to segment data for each process kernel.epoch=6 #set the number of epochs to train kernel.batch=32 #set the batch size kernel.PO=2 #use PO2 algorithm for parallel optimization @@ -172,7 +172,7 @@ nn=n.nn() #create neural network object nn.build() #build the network structure kernel=k.kernel(nn) #create kernel object with the network kernel.process=7 #set the number of processes to train -kernel.data_segment_flag=True #set the flag to segment data +kernel.data_segment_flag=True #set the flag to segment data for each process kernel.epoch=6 #set the number of epochs to train kernel.batch=32 #set the batch size kernel.PO=3 #use PO3 algorithm for parallel optimization @@ -197,7 +197,7 @@ nn=n.nn() #create neural network object nn.build() #build the network structure kernel=k.kernel(nn) #create kernel object with the network kernel.process=7 #set the number of processes to train -kernel.data_segment_flag=True #set the flag to segment data +kernel.data_segment_flag=True #set the flag to segment data for each process kernel.epoch=6 #set the number of epochs to train kernel.batch=32 #set the batch size kernel.PO=3 #use PO3 algorithm for parallel optimization @@ -224,7 +224,7 @@ nn=n.nn() #create neural network object nn.build() #build the network structure kernel=k.kernel(nn) #create kernel object with the network kernel.process=7 #set the number of processes to train -kernel.data_segment_flag=True #set the flag to segment data +kernel.data_segment_flag=True #set the flag to segment data for each process kernel.epoch=6 #set the number of epochs to train kernel.batch=32 #set the batch size kernel.PO=3 #use PO3 algorithm for parallel optimization @@ -247,7 +247,7 @@ mnist=tf.keras.datasets.mnist #load mnist dataset x_train,x_test =x_train/255.0,x_test/255.0 #normalize data kernel=k.kernel() #create kernel object without a network kernel.process=7 #set the number of processes to train -kernel.data_segment_flag=True #set the flag to segment data +kernel.data_segment_flag=True #set the flag to segment data for each process kernel.epoch=1 #set the number of epochs to train kernel.batch=32 #set the batch size kernel.PO=3 #use PO3 algorithm for parallel optimization @@ -273,7 +273,7 @@ nn.build() #build the network structure kernel=k.kernel(nn) #create kernel object with the network kernel.process=7 #set the number of processes to train kernel.process_t=3 #set the number of processes to test -kernel.data_segment_flag=True #set the flag to segment data +kernel.data_segment_flag=True #set the flag to segment data for each process kernel.epoch=6 #set the number of epochs to train kernel.batch=32 #set the batch size kernel.PO=3 #use PO3 algorithm for parallel optimization @@ -300,7 +300,7 @@ kernel=k.kernel(nn) #create kernel object with the network kernel.stop=True #set the flag to stop training when a condition is met kernel.end_loss=0.7 #set the condition to stop training when the loss is less than 0.7 kernel.process=7 #set the number of processes to train -kernel.data_segment_flag=True #set the flag to segment data +kernel.data_segment_flag=True #set the flag to segment data for each process kernel.epoch=6 #set the number of epochs to train kernel.batch=32 #set the batch size kernel.PO=2 #use PO2 algorithm for parallel optimization @@ -392,7 +392,7 @@ x_train,x_test =x_train/255.0,x_test/255.0 #normalize data nn=n.nn() #create neural network object nn.build() #build the network structure test=t.parallel_test(nn,x_test,y_test,6,32) #create parallel test object with the network, the test data, the number of processes and the batch size -test.segment_data() #segment data +test.segment_data() #segment data for each process for p in range(6): #loop over the processes Process(target=test.test).start() #start each process with the test function loss=test.loss_acc() #calculate the loss and accuracy of the test From c69ebd16425a41c27366755635e780a8a72e358e Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Sun, 16 Jul 2023 09:24:23 +0800 Subject: [PATCH 053/337] Delete Note 7.0 documentation/RL/neural network/tensorflow/pool net/thread directory --- .../tensorflow/pool net/thread/DDPG.py | 84 ------------------- .../tensorflow/pool net/thread/DQN.py | 46 ---------- .../tensorflow/pool net/thread/DQN_m.py | 48 ----------- .../tensorflow/pool net/thread/DQN_mtpr.py | 59 ------------- 4 files changed, 237 deletions(-) delete mode 100644 Note 7.0 documentation/RL/neural network/tensorflow/pool net/thread/DDPG.py delete mode 100644 Note 7.0 documentation/RL/neural network/tensorflow/pool net/thread/DQN.py delete mode 100644 Note 7.0 documentation/RL/neural network/tensorflow/pool net/thread/DQN_m.py delete mode 100644 Note 7.0 documentation/RL/neural network/tensorflow/pool net/thread/DQN_mtpr.py diff --git a/Note 7.0 documentation/RL/neural network/tensorflow/pool net/thread/DDPG.py b/Note 7.0 documentation/RL/neural network/tensorflow/pool net/thread/DDPG.py deleted file mode 100644 index 6377d12f..00000000 --- a/Note 7.0 documentation/RL/neural network/tensorflow/pool net/thread/DDPG.py +++ /dev/null @@ -1,84 +0,0 @@ -import tensorflow as tf -import numpy as np -import gym - - -class actor: - def __init__(self,state_dim,hidden_dim,action_dim,action_bound): - self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) - self.bias1=tf.Variable(tf.random.normal([hidden_dim])) - self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) - self.bias2=tf.Variable(tf.random.normal([action_dim])) - self.action_bound=action_bound - self.param=[self.weight1,self.bias1,self.weight2,self.bias2] - - - def fp(self,x): - x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) - return tf.nn.tanh(tf.matmul(x,self.weight2)+self.bias2)*self.action_bound - - -class critic: - def __init__(self,state_dim,hidden_dim,action_dim): - self.weight1=tf.Variable(tf.random.normal([state_dim+action_dim,hidden_dim])) - self.bias1=tf.Variable(tf.random.normal([hidden_dim])) - self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) - self.bias2=tf.Variable(tf.random.normal([action_dim])) - self.param=[self.weight1,self.bias1,self.weight2,self.bias2] - - - def fp(self,x,a): - cat=tf.concat([x,a],axis=1) - x=tf.nn.relu(tf.matmul(cat,self.weight1)+self.bias1) - return tf.matmul(x,self.weight2)+self.bias2 - - -class DDPG: - def __init__(self,hidden_dim,sigma,gamma,tau,actor_lr,critic_lr): - self.genv=[gym.make('Pendulum-v0') for _ in range(5)] - state_dim=self.genv[0].observation_space.shape[0] - action_dim=self.genv[0].action_space.shape[0] - action_bound=self.genv[0].action_space.high[0] - self.actor=actor(state_dim,hidden_dim,action_dim,action_bound) - self.critic=critic(state_dim,hidden_dim,action_dim) - self.target_actor=actor(state_dim,hidden_dim,action_dim,action_bound) - self.target_critic=critic(state_dim,hidden_dim,action_dim) - self.actor_param=self.actor.param - self.critic_param=self.critic.param - self.target_actor.param=self.actor_param.copy() - self.target_critic.param=self.critic_param.copy() - self.param=[self.actor_param,self.critic_param] - self.sigma=sigma - self.gamma=gamma - self.tau=tau - self.opt=tf.keras.optimizers.Adam() - - - def noise(self): - return np.random.normal(scale=self.sigma) - - - def env(self,a=None,t=None,initial=None): - if initial==True: - state=self.genv[t].reset(seed=0) - return state - else: - next_state,reward,done,_=self.genv[t].step(a) - return next_state,reward,done - - - def loss(self,s,a,next_s,r,d): - a=tf.expand_dims(a,axis=1) - next_q_value=self.target_critic.fp(next_s,self.target_actor.fp(next_s)) - q_target=r+self.gamma*next_q_value*(1-d) - actor_loss=-tf.reduce_mean(self.critic.fp(s,self.actor.fp(s))) - critic_loss=tf.reduce_mean((self.critic.fp(s,a)-q_target)**2) - return [actor_loss,critic_loss] - - - def update_param(self): - for target_param,param in zip(self.target_actor.param,self.actor.param): - target_param=target_param*(1.0-self.tau)+param*self.tau - for target_param,param in zip(self.target_critic.param,self.critic.param): - target_param=target_param*(1.0-self.tau)+param*self.tau - return \ No newline at end of file diff --git a/Note 7.0 documentation/RL/neural network/tensorflow/pool net/thread/DQN.py b/Note 7.0 documentation/RL/neural network/tensorflow/pool net/thread/DQN.py deleted file mode 100644 index d813f782..00000000 --- a/Note 7.0 documentation/RL/neural network/tensorflow/pool net/thread/DQN.py +++ /dev/null @@ -1,46 +0,0 @@ -import tensorflow as tf -import gym - - -class Qnet: - def __init__(self,state_dim,hidden_dim,action_dim): - self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) - self.bias1=tf.Variable(tf.random.normal([hidden_dim])) - self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) - self.bias2=tf.Variable(tf.random.normal([action_dim])) - self.param=[self.weight1,self.bias1,self.weight2,self.bias2] - - def fp(self,x): #forward propagation function,kernel uses it for forward propagation. - x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) - return tf.matmul(x,self.weight2)+self.bias2 - - -class DQN: - def __init__(self,state_dim,hidden_dim,action_dim): - self.nn=Qnet(state_dim,hidden_dim,action_dim) - self.target_q_net=Qnet(state_dim,hidden_dim,action_dim) - self.param=self.nn.param #parameter list,kernel uses it list for backpropagation. - self.opt=tf.keras.optimizers.Adam() #optimizer,kernel uses it to optimize. - self.genv=[gym.make('CartPole-v0') for _ in range(5)] #create environment. - - - def env(self,a=None,t=None,initial=None): #environment function,kernel uses it to interact with the environment. - if initial==True: - state=self.genv[t].reset(seed=0) - return state - else: - next_state,reward,done,_=self.genv[t].step(a) - return next_state,reward,done - - - def loss(self,s,a,next_s,r,d): #loss functino,kernel uses it to calculate loss. - a=tf.expand_dims(a,axis=1) - q_value=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) - next_q_value=tf.reduce_max(self.target_q_net.fp(next_s),axis=1) - target=r+0.98*next_q_value*(1-d) - return tf.reduce_mean((q_value-target)**2) - - - def update_param(self): #update function,kernel uses it to update parameter. - self.target_q_net.param=self.param.copy() - return \ No newline at end of file diff --git a/Note 7.0 documentation/RL/neural network/tensorflow/pool net/thread/DQN_m.py b/Note 7.0 documentation/RL/neural network/tensorflow/pool net/thread/DQN_m.py deleted file mode 100644 index 9639e15c..00000000 --- a/Note 7.0 documentation/RL/neural network/tensorflow/pool net/thread/DQN_m.py +++ /dev/null @@ -1,48 +0,0 @@ -import tensorflow as tf -import gym - - -class Qnet: - def __init__(self,state_dim,hidden_dim,action_dim): - self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) - self.bias1=tf.Variable(tf.random.normal([hidden_dim])) - self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) - self.bias2=tf.Variable(tf.random.normal([action_dim])) - self.param=[self.weight1,self.bias1,self.weight2,self.bias2] - - def fp(self,x): - x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) - return tf.matmul(x,self.weight2)+self.bias2 - - -class DQN: - def __init__(self,state_dim,hidden_dim,action_dim): - self.nn=Qnet(state_dim,hidden_dim,action_dim) - self.target_q_net=Qnet(state_dim,hidden_dim,action_dim) - self.param=self.nn.param - self.opt=tf.keras.optimizers.Adam() - self.genv=[gym.make('CartPole-v0') for _ in range(5)] - self.row=2 - self.rank=2 - - - def env(self,a=None,t=None,initial=None): - if initial==True: - state=self.genv[t].reset(seed=0) - return state - else: - next_state,reward,done,_=self.genv[t].step(a) - return next_state,reward,done - - - def loss(self,s,a,next_s,r,d): - a=tf.expand_dims(a,axis=1) - q_value=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) - next_q_value=tf.reduce_max(self.target_q_net.fp(next_s),axis=1) - target=r+0.98*next_q_value*(1-d) - return tf.reduce_mean((q_value-target)**2) - - - def update_param(self): - self.target_q_net.param=self.param.copy() - return \ No newline at end of file diff --git a/Note 7.0 documentation/RL/neural network/tensorflow/pool net/thread/DQN_mtpr.py b/Note 7.0 documentation/RL/neural network/tensorflow/pool net/thread/DQN_mtpr.py deleted file mode 100644 index 107e4e17..00000000 --- a/Note 7.0 documentation/RL/neural network/tensorflow/pool net/thread/DQN_mtpr.py +++ /dev/null @@ -1,59 +0,0 @@ -import tensorflow as tf -import gym -import Note.create.RL.rl.prioritized_replay as pr - - -class Qnet: - def __init__(self,state_dim,hidden_dim,action_dim): - self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) - self.bias1=tf.Variable(tf.random.normal([hidden_dim])) - self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) - self.bias2=tf.Variable(tf.random.normal([action_dim])) - self.param=[self.weight1,self.bias1,self.weight2,self.bias2] - - def fp(self,x): - x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) - return tf.matmul(x,self.weight2)+self.bias2 - - -class DQN: - def __init__(self,state_dim,hidden_dim,action_dim): - self.nn=Qnet(state_dim,hidden_dim,action_dim) - self.target_q_net=Qnet(state_dim,hidden_dim,action_dim) - self.param=self.nn.param - self.pr=pr.pr_mt() - self.initial_TD=7 - self._epsilon=0.0007 - self.alpha=0.7 - self.opt=tf.keras.optimizers.Adam() - self.genv=[gym.make('CartPole-v0') for _ in range(5)] - - - def env(self,a=None,t=None,initial=None): - if initial==True: - state=self.genv[t].reset(seed=0) - return state - else: - next_state,reward,done,_=self.genv[t].step(a) - return next_state,reward,done - - - def data_func(self,state_pool,action_pool,next_state_pool,reward_pool,done_pool,batch,t): - s,a,next_s,r,d=self.pr.sample(state_pool,action_pool,next_state_pool,reward_pool,done_pool,self.epsilon,self.alpha,batch,t) - return s,a,next_s,r,d - - - def loss(self,s,a,next_s,r,d,t): - a=tf.expand_dims(a,axis=1) - q_value=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) - next_q_value=tf.reduce_max(self.target_q_net.fp(next_s),axis=1) - target=r+0.98*next_q_value*(1-d) - target=tf.expand_dims(target,axis=1) - TD=target-q_value - self.pr.update_TD(TD,t) - return tf.reduce_mean(TD**2) - - - def update_param(self): - self.target_q_net.param=self.param.copy() - return \ No newline at end of file From 2b64b9c8e7d01c8cdbd963737fedab75b7c6b0e6 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Sun, 16 Jul 2023 09:38:46 +0800 Subject: [PATCH 054/337] Update DQN.py --- .../RL/neural network/tensorflow/DQN.py | 72 +++++++++---------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/Note 7.0 documentation/RL/neural network/tensorflow/DQN.py b/Note 7.0 documentation/RL/neural network/tensorflow/DQN.py index 89087e26..9b4adcde 100644 --- a/Note 7.0 documentation/RL/neural network/tensorflow/DQN.py +++ b/Note 7.0 documentation/RL/neural network/tensorflow/DQN.py @@ -1,46 +1,46 @@ -import tensorflow as tf -import gym +import tensorflow as tf # import TensorFlow library +import gym # import OpenAI Gym library -class Qnet: - def __init__(self,state_dim,hidden_dim,action_dim): - self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) - self.bias1=tf.Variable(tf.random.normal([hidden_dim])) - self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) - self.bias2=tf.Variable(tf.random.normal([action_dim])) - self.param=[self.weight1,self.bias1,self.weight2,self.bias2] - - def fp(self,x): #forward propagation function,kernel uses it for forward propagation. - x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) - return tf.matmul(x,self.weight2)+self.bias2 - - -class DQN: - def __init__(self,state_dim,hidden_dim,action_dim): - self.nn=Qnet(state_dim,hidden_dim,action_dim) - self.target_q_net=Qnet(state_dim,hidden_dim,action_dim) - self.param=self.nn.param #parameter list,kernel uses it list for backpropagation. - self.opt=tf.keras.optimizers.Adam() #optimizer,kernel uses it to optimize. - self.genv=gym.make('CartPole-v0') #create environment. - - - def env(self,a=None,initial=None): #environment function,kernel uses it to interact with the environment. - if initial==True: +class Qnet: # define a class for the Q-network + def __init__(self,state_dim,hidden_dim,action_dim): # initialize the network with state dimension, hidden dimension and action dimension + self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) # create a weight matrix for the first layer + self.bias1=tf.Variable(tf.random.normal([hidden_dim])) # create a bias vector for the first layer + self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) # create a weight matrix for the second layer + self.bias2=tf.Variable(tf.random.normal([action_dim])) # create a bias vector for the second layer + self.param=[self.weight1,self.bias1,self.weight2,self.bias2] # store the network parameters in a list + + def fp(self,x): # forward propagation function, kernel uses it for forward propagation + x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) # apply the first layer with ReLU activation + return tf.matmul(x,self.weight2)+self.bias2 # apply the second layer and return the output + + +class DQN: # define a class for the DQN agent + def __init__(self,state_dim,hidden_dim,action_dim): # initialize the agent with state dimension, hidden dimension and action dimension + self.nn=Qnet(state_dim,hidden_dim,action_dim) # create a Q-network for the agent + self.target_q_net=Qnet(state_dim,hidden_dim,action_dim) # create a target Q-network for the agent + self.param=self.nn.param # parameter list, kernel uses it list for backpropagation + self.opt=tf.keras.optimizers.Adam() # optimizer, kernel uses it to optimize + self.genv=gym.make('CartPole-v0') # create environment + + + def env(self,a=None,initial=None): # environment function, kernel uses it to interact with the environment + if initial==True: # if initial is True, reset the environment and return the initial state state=self.genv.reset(seed=0) return state - else: + else: # otherwise, take an action and return the next state, reward and done flag next_state,reward,done,_=self.genv.step(a) return next_state,reward,done - def loss(self,s,a,next_s,r,d): #loss functino,kernel uses it to calculate loss. - a=tf.expand_dims(a,axis=1) - q_value=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) - next_q_value=tf.reduce_max(self.target_q_net.fp(next_s),axis=1) - target=r+0.98*next_q_value*(1-d) - return tf.reduce_mean((q_value-target)**2) + def loss(self,s,a,next_s,r,d): # loss function, kernel uses it to calculate loss + a=tf.expand_dims(a,axis=1) # expand the action vector to match the Q-value matrix shape + q_value=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) # get the Q-value for the selected action + next_q_value=tf.reduce_max(self.target_q_net.fp(next_s),axis=1) # get the maximum Q-value for the next state from the target network + target=r+0.98*next_q_value*(1-d) # calculate the target value using Bellman equation with discount factor 0.98 + return tf.reduce_mean((q_value-target)**2) # return the mean squared error between Q-value and target value - def update_param(self): #update function,kernel uses it to update parameter. - self.target_q_net.param=self.param.copy() - return \ No newline at end of file + def update_param(self): # update function, kernel uses it to update parameter + self.target_q_net.param=self.param.copy() # copy the parameters from the Q-network to the target network + return From fa7a550d1bf767e19c9fe762f8b79bf7ceeb98f3 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Sun, 16 Jul 2023 09:39:04 +0800 Subject: [PATCH 055/337] Update DQN.py --- .../neural network/tensorflow/pool net/DQN.py | 78 +++++++++---------- 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/Note 7.0 documentation/RL/neural network/tensorflow/pool net/DQN.py b/Note 7.0 documentation/RL/neural network/tensorflow/pool net/DQN.py index 1ffc1805..4bd4ba28 100644 --- a/Note 7.0 documentation/RL/neural network/tensorflow/pool net/DQN.py +++ b/Note 7.0 documentation/RL/neural network/tensorflow/pool net/DQN.py @@ -1,52 +1,52 @@ -import tensorflow as tf -import gym -import Note.nn.process.optimizer as o +import tensorflow as tf # import TensorFlow library +import gym # import OpenAI Gym library +import Note.nn.process.optimizer as o # import Note's optimizer module -class Qnet: - def __init__(self,state_dim,hidden_dim,action_dim): - self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) - self.bias1=tf.Variable(tf.random.normal([hidden_dim])) - self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) - self.bias2=tf.Variable(tf.random.normal([action_dim])) - self.param=[self.weight1,self.bias1,self.weight2,self.bias2] - - def fp(self,x): - x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) - return tf.matmul(x,self.weight2)+self.bias2 - - -class DQN: - def __init__(self,state_dim,hidden_dim,action_dim): - self.nn=Qnet(state_dim,hidden_dim,action_dim) - self.target_q_net=Qnet(state_dim,hidden_dim,action_dim) - self.param=self.nn.param - self.optimizer=o.Momentum(0.07,0.7) - self.genv=[gym.make('CartPole-v0') for _ in range(5)] - - - def env(self,a=None,p=None,initial=None): - if initial==True: +class Qnet: # define a class for the Q-network + def __init__(self,state_dim,hidden_dim,action_dim): # initialize the network with state dimension, hidden dimension and action dimension + self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) # create a weight matrix for the first layer + self.bias1=tf.Variable(tf.random.normal([hidden_dim])) # create a bias vector for the first layer + self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) # create a weight matrix for the second layer + self.bias2=tf.Variable(tf.random.normal([action_dim])) # create a bias vector for the second layer + self.param=[self.weight1,self.bias1,self.weight2,self.bias2] # store the network parameters in a list + + def fp(self,x): # forward propagation function, kernel uses it for forward propagation + x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) # apply the first layer with ReLU activation + return tf.matmul(x,self.weight2)+self.bias2 # apply the second layer and return the output + + +class DQN: # define a class for the DQN agent + def __init__(self,state_dim,hidden_dim,action_dim): # initialize the agent with state dimension, hidden dimension and action dimension + self.nn=Qnet(state_dim,hidden_dim,action_dim) # create a Q-network for the agent + self.target_q_net=Qnet(state_dim,hidden_dim,action_dim) # create a target Q-network for the agent + self.param=self.nn.param # parameter list, kernel uses it list for backpropagation + self.optimizer=o.Momentum(0.07,0.7) # optimizer, kernel uses it to optimize. Here we use a custom momentum optimizer with learning rate 0.07 and momentum 0.7 + self.genv=[gym.make('CartPole-v0') for _ in range(5)] # create a list of 5 environments + + + def env(self,a=None,p=None,initial=None): # environment function, kernel uses it to interact with the environment + if initial==True: # if initial is True, reset the environment with index p and return the initial state state=self.genv[p].reset(seed=0) return state - else: + else: # otherwise, take an action and return the next state, reward and done flag from the environment with index p next_state,reward,done,_=self.genv[p].step(a) return next_state,reward,done - def loss(self,s,a,next_s,r,d): - a=tf.expand_dims(a,axis=1) - q_value=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) - next_q_value=tf.reduce_max(self.target_q_net.fp(next_s),axis=1) - target=r+0.98*next_q_value*(1-d) - return tf.reduce_mean((q_value-target)**2) + def loss(self,s,a,next_s,r,d): # loss function, kernel uses it to calculate loss + a=tf.expand_dims(a,axis=1) # expand the action vector to match the Q-value matrix shape + q_value=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) # get the Q-value for the selected action + next_q_value=tf.reduce_max(self.target_q_net.fp(next_s),axis=1) # get the maximum Q-value for the next state from the target network + target=r+0.98*next_q_value*(1-d) # calculate the target value using Bellman equation with discount factor 0.98 + return tf.reduce_mean((q_value-target)**2) # return the mean squared error between Q-value and target value - def update_param(self): - self.target_q_net.param=self.param.copy() + def update_param(self): # update function, kernel uses it to update parameter + self.target_q_net.param=self.param.copy() # copy the parameters from the Q-network to the target network return - def opt(self,gradient): - param=self.optimizer.opt(gradient,self.param) - return param \ No newline at end of file + def opt(self,gradient): # optimization function, kernel uses it to optimize parameter + param=self.optimizer.opt(gradient,self.param) # apply the custom momentum optimizer to update the parameters using the gradient + return param From 65761ed9a1e1f008f85f858c0fc8e51406084d65 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Sun, 16 Jul 2023 09:42:04 +0800 Subject: [PATCH 056/337] Update nn_ol.py --- .../DL/neural network/tensorflow/nn_ol.py | 52 +++++++++---------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/nn_ol.py b/Note 7.0 documentation/DL/neural network/tensorflow/nn_ol.py index 0f3150e0..7bacada0 100644 --- a/Note 7.0 documentation/DL/neural network/tensorflow/nn_ol.py +++ b/Note 7.0 documentation/DL/neural network/tensorflow/nn_ol.py @@ -1,37 +1,37 @@ -import tensorflow as tf -import numpy as np +import tensorflow as tf # import TensorFlow library +import numpy as np # import NumPy library -#This is an example of online training. -class nn: - def __init__(self,data,labels): - self.train_data=data - self.train_labels=labels - self.model=tf.keras.models.Sequential([ - tf.keras.layers.Flatten(input_shape=(28, 28)), - tf.keras.layers.Dense(128,activation='relu'), - tf.keras.layers.Dropout(0.2), - tf.keras.layers.Dense(10) +# This is an example of online training. +class nn: # define a class for the neural network + def __init__(self,data,labels): # initialize the network with data and labels + self.train_data=data # store the training data + self.train_labels=labels # store the training labels + self.model=tf.keras.models.Sequential([ # create a sequential model with four layers + tf.keras.layers.Flatten(input_shape=(28, 28)), # flatten the input to a vector of 28*28 elements + tf.keras.layers.Dense(128,activation='relu'), # add a dense layer with 128 units and ReLU activation + tf.keras.layers.Dropout(0.2), # add a dropout layer with 0.2 dropout rate to prevent overfitting + tf.keras.layers.Dense(10) # add a dense layer with 10 units for the output logits ]) - self.param=self.model.weights #parameter list,kernel uses it list for backpropagation. - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.opt=tf.keras.optimizers.Adam() #optimizer,kernel uses it to optimize. - self.train_loss_list=[] - self.counter=0 #counter - self.max_length=1000 + self.param=self.model.weights # parameter list, kernel uses it list for backpropagation + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) # loss object, kernel uses it to calculate loss. Here we use sparse categorical crossentropy with logits as output + self.opt=tf.keras.optimizers.Adam() # optimizer, kernel uses it to optimize. Here we use Adam optimizer + self.train_loss_list=[] # a list to store the training loss values + self.counter=0 # counter to keep track of the number of online updates + self.max_length=1000 # maximum length of train_loss_list - def online(self): #This is simulative online function. - index=np.random.choice(60000,size=[32]) - if self.counter==10000: + def online(self): # This is simulative online function, kernel uses it to get online data and labels + index=np.random.choice(60000,size=[32]) # randomly sample 32 indices from 0 to 59999 + if self.counter==10000: # if the counter reaches 10000, stop the online training return 'stop' - else: + else: # otherwise, return the data and labels corresponding to the sampled indices return [self.train_data[index],self.train_labels[index]] - def fp(self,data): #forward propagation function,kernel uses it for forward propagation. - output=self.model(data) + def fp(self,data): # forward propagation function, kernel uses it for forward propagation + output=self.model(data) # pass the data through the model and get the output logits return output - def loss(self,output,labels): #loss functino,kernel uses it to calculate loss. - return self.loss_object(labels,output) + def loss(self,output,labels): # loss function, kernel uses it to calculate loss + return self.loss_object(labels,output) # return the sparse categorical crossentropy loss between labels and output From e877b2b25e959ae05346b3028f9d0f4e7cdb32d6 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Sun, 16 Jul 2023 09:42:55 +0800 Subject: [PATCH 057/337] Update DDPG.py --- .../RL/neural network/tensorflow/DDPG.py | 124 +++++++++--------- 1 file changed, 62 insertions(+), 62 deletions(-) diff --git a/Note 7.0 documentation/RL/neural network/tensorflow/DDPG.py b/Note 7.0 documentation/RL/neural network/tensorflow/DDPG.py index 3bd32083..9f4b243a 100644 --- a/Note 7.0 documentation/RL/neural network/tensorflow/DDPG.py +++ b/Note 7.0 documentation/RL/neural network/tensorflow/DDPG.py @@ -1,84 +1,84 @@ -import tensorflow as tf -import numpy as np -import gym +import tensorflow as tf # import TensorFlow library +import numpy as np # import NumPy library +import gym # import OpenAI Gym library -class actor: - def __init__(self,state_dim,hidden_dim,action_dim,action_bound): - self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) - self.bias1=tf.Variable(tf.random.normal([hidden_dim])) - self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) - self.bias2=tf.Variable(tf.random.normal([action_dim])) - self.action_bound=action_bound - self.param=[self.weight1,self.bias1,self.weight2,self.bias2] +class actor: # define a class for the actor network + def __init__(self,state_dim,hidden_dim,action_dim,action_bound): # initialize the network with state dimension, hidden dimension, action dimension and action bound + self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) # create a weight matrix for the first layer + self.bias1=tf.Variable(tf.random.normal([hidden_dim])) # create a bias vector for the first layer + self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) # create a weight matrix for the second layer + self.bias2=tf.Variable(tf.random.normal([action_dim])) # create a bias vector for the second layer + self.action_bound=action_bound # store the action bound + self.param=[self.weight1,self.bias1,self.weight2,self.bias2] # store the network parameters in a list - def fp(self,x): - x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) - return tf.nn.tanh(tf.matmul(x,self.weight2)+self.bias2)*self.action_bound + def fp(self,x): # forward propagation function, kernel uses it for forward propagation + x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) # apply the first layer with ReLU activation + return tf.nn.tanh(tf.matmul(x,self.weight2)+self.bias2)*self.action_bound # apply the second layer with tanh activation and scale it by the action bound -class critic: - def __init__(self,state_dim,hidden_dim,action_dim): - self.weight1=tf.Variable(tf.random.normal([state_dim+action_dim,hidden_dim])) - self.bias1=tf.Variable(tf.random.normal([hidden_dim])) - self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) - self.bias2=tf.Variable(tf.random.normal([action_dim])) - self.param=[self.weight1,self.bias1,self.weight2,self.bias2] +class critic: # define a class for the critic network + def __init__(self,state_dim,hidden_dim,action_dim): # initialize the network with state dimension, hidden dimension and action dimension + self.weight1=tf.Variable(tf.random.normal([state_dim+action_dim,hidden_dim])) # create a weight matrix for the first layer + self.bias1=tf.Variable(tf.random.normal([hidden_dim])) # create a bias vector for the first layer + self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) # create a weight matrix for the second layer + self.bias2=tf.Variable(tf.random.normal([action_dim])) # create a bias vector for the second layer + self.param=[self.weight1,self.bias1,self.weight2,self.bias2] # store the network parameters in a list - def fp(self,x,a): - cat=tf.concat([x,a],axis=1) - x=tf.nn.relu(tf.matmul(cat,self.weight1)+self.bias1) - return tf.matmul(x,self.weight2)+self.bias2 + def fp(self,x,a): # forward propagation function, kernel uses it for forward propagation + cat=tf.concat([x,a],axis=1) # concatenate the state and action vectors along the first axis + x=tf.nn.relu(tf.matmul(cat,self.weight1)+self.bias1) # apply the first layer with ReLU activation + return tf.matmul(x,self.weight2)+self.bias2 # apply the second layer and return the output -class DDPG: - def __init__(self,hidden_dim,sigma,gamma,tau,actor_lr,critic_lr): - self.genv=gym.make('Pendulum-v0') - state_dim=self.genv.observation_space.shape[0] - action_dim=self.genv.action_space.shape[0] - action_bound=self.genv.action_space.high[0] - self.actor=actor(state_dim,hidden_dim,action_dim,action_bound) - self.critic=critic(state_dim,hidden_dim,action_dim) - self.target_actor=actor(state_dim,hidden_dim,action_dim,action_bound) - self.target_critic=critic(state_dim,hidden_dim,action_dim) - self.actor_param=self.actor.param - self.critic_param=self.critic.param - self.target_actor.param=self.actor_param.copy() - self.target_critic.param=self.critic_param.copy() - self.param=[self.actor_param,self.critic_param] - self.sigma=sigma - self.gamma=gamma - self.tau=tau - self.opt=tf.keras.optimizers.Adam() +class DDPG: # define a class for the DDPG agent + def __init__(self,hidden_dim,sigma,gamma,tau,actor_lr,critic_lr): # initialize the agent with hidden dimension, noise scale, discount factor, soft update factor, actor learning rate and critic learning rate + self.genv=gym.make('Pendulum-v0') # create environment + state_dim=self.genv.observation_space.shape[0] # get the state dimension from the environment observation space + action_dim=self.genv.action_space.shape[0] # get the action dimension from the environment action space + action_bound=self.genv.action_space.high[0] # get the action bound from the environment action space high value (assume symmetric) + self.actor=actor(state_dim,hidden_dim,action_dim,action_bound) # create an actor network for the agent + self.critic=critic(state_dim,hidden_dim,action_dim) # create a critic network for the agent + self.target_actor=actor(state_dim,hidden_dim,action_dim,action_bound) # create a target actor network for the agent + self.target_critic=critic(state_dim,hidden_dim,action_dim) # create a target critic network for the agent + self.actor_param=self.actor.param # parameter list of actor network, kernel uses it list for backpropagation + self.critic_param=self.critic.param # parameter list of critic network, kernel uses it list for backpropagation + self.target_actor.param=self.actor_param.copy() # copy the parameters from actor network to target actor network + self.target_critic.param=self.critic_param.copy() # copy the parameters from critic network to target critic network + self.param=[self.actor_param,self.critic_param] # parameter list of both networks, kernel uses it list for backpropagation + self.sigma=sigma # noise scale + self.gamma=gamma # discount factor + self.tau=tau # soft update factor + self.opt=tf.keras.optimizers.Adam() # optimizer, kernel uses it to optimize. Here we use Adam optimizer - def noise(self): - return np.random.normal(scale=self.sigma) + def noise(self): # noise function, kernel uses it to generate exploration noise + return np.random.normal(scale=self.sigma) # return a random sample from a normal distribution with zero mean and sigma scale - def env(self,a=None,initial=None): - if initial==True: + def env(self,a=None,initial=None): # environment function, kernel uses it to interact with the environment + if initial==True: # if initial is True, reset the environment and return the initial state state=self.genv.reset(seed=0) return state - else: + else: # otherwise, take an action and return the next state, reward and done flag next_state,reward,done,_=self.genv.step(a) return next_state,reward,done - def loss(self,s,a,next_s,r,d): - a=tf.expand_dims(a,axis=1) - next_q_value=self.target_critic.fp(next_s,self.target_actor.fp(next_s)) - q_target=r+self.gamma*next_q_value*(1-d) - actor_loss=-tf.reduce_mean(self.critic.fp(s,self.actor.fp(s))) - critic_loss=tf.reduce_mean((self.critic.fp(s,a)-q_target)**2) - return [actor_loss,critic_loss] + def loss(self,s,a,next_s,r,d): # loss function, kernel uses it to calculate loss + a=tf.expand_dims(a,axis=1) # expand the action vector to match the Q-value matrix shape + next_q_value=self.target_critic.fp(next_s,self.target_actor.fp(next_s)) # get the Q-value for the next state and action from the target critic network + q_target=r+self.gamma*next_q_value*(1-d) # calculate the target value using Bellman equation with discount factor gamma + actor_loss=-tf.reduce_mean(self.critic.fp(s,self.actor.fp(s))) # calculate the actor loss as the negative mean of the Q-value for the current state and action from the critic network + critic_loss=tf.reduce_mean((self.critic.fp(s,a)-q_target)**2) # calculate the critic loss as the mean squared error between Q-value and target value + return [actor_loss,critic_loss] # return a list of actor loss and critic loss - def update_param(self): - for target_param,param in zip(self.target_actor.param,self.actor.param): - target_param=target_param*(1.0-self.tau)+param*self.tau - for target_param,param in zip(self.target_critic.param,self.critic.param): - target_param=target_param*(1.0-self.tau)+param*self.tau - return \ No newline at end of file + def update_param(self): # update function, kernel uses it to update parameter + for target_param,param in zip(self.target_actor.param,self.actor.param): # for each pair of parameters in target actor network and actor network + target_param=target_param*(1.0-self.tau)+param*self.tau # update the target parameter using soft update with factor tau + for target_param,param in zip(self.target_critic.param,self.critic.param): # for each pair of parameters in target critic network and critic network + target_param=target_param*(1.0-self.tau)+param*self.tau # update the target parameter using soft update with factor tau + return From f1e49bb5573c48aa2f7faa6aace7f685df1f5352 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Sun, 16 Jul 2023 09:57:24 +0800 Subject: [PATCH 058/337] Add files via upload --- .../tensorflow/process/nn_device.py | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/process/nn_device.py diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/process/nn_device.py b/Note 7.0 documentation/DL/neural network/tensorflow/process/nn_device.py new file mode 100644 index 00000000..f390259d --- /dev/null +++ b/Note 7.0 documentation/DL/neural network/tensorflow/process/nn_device.py @@ -0,0 +1,39 @@ +import tensorflow as tf # import TensorFlow library +import Note.nn.layer.dense as d # import Note's dense layer module +from Note.nn.layer.flatten import flatten # import Note's flatten layer function +from Note.nn.process.optimizer import Momentum # import Note's momentum optimizer module + + +class nn: # A neural network class example, allocate device for multiple threads + def __init__(self): # initialize the network + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) # loss object, kernel uses it to calculate loss. Here we use sparse categorical crossentropy with logits as output + self.optimizer=Momentum(0.07,0.7) # optimizer, kernel uses it to optimize. Here we use a custom momentum optimizer with learning rate 0.07 and momentum 0.7 + self.device_table={0:'GPU:0',1:'GPU:0',2:'GPU:0',3:'GPU:1',4:'GPU:1',5:'GPU:1',6:'GPU:2'} # a dictionary to map process index to device name + self.info='example' # some information about the network + + + def build(self): # build function, kernel uses it to create the network layers + # Create two dense layers with relu and linear activations + self.layer1=d.dense([784,128],activation='relu') # the first layer with 784 input units and 128 output units and ReLU activation + self.layer2=d.dense([128,10]) # the second layer with 128 input units and 10 output units and linear activation + # Store the parameters of the layers in a list + self.param=[self.layer1.param,self.layer2.param] # parameter list of both layers, kernel uses it list for backpropagation + return + + + def fp(self,data,p): # forward propagation function, kernel uses it for forward propagation + with tf.device(self.device_table[p]): # assign the device according to the process index p + data=flatten(data) # flatten the data to a vector of 784 elements + output1=self.layer1.output(data) # pass the data through the first layer and get the output + output2=self.layer2.output(output1) # pass the output of the first layer through the second layer and get the final output logits + return output2 + + + def loss(self,output,labels): # loss function, kernel uses it to calculate loss + return self.loss_object(labels,output) # return the mean softmax cross entropy loss between labels and output + + + def opt(self,gradient): # optimization function, kernel uses it to optimize parameter + # Perform optimization on the parameters using the gradient + param=self.optimizer.opt(gradient,self.param) # apply the Note's momentum optimizer to update the parameters using the gradient + return param \ No newline at end of file From e3ecf76082ac7bd46f70ea6b4200aa02ef2ce14a Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Wed, 19 Jul 2023 08:36:43 +0800 Subject: [PATCH 059/337] Update nn_device.py --- .../DL/neural network/tensorflow/process/nn_device.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/process/nn_device.py b/Note 7.0 documentation/DL/neural network/tensorflow/process/nn_device.py index f390259d..e1720a74 100644 --- a/Note 7.0 documentation/DL/neural network/tensorflow/process/nn_device.py +++ b/Note 7.0 documentation/DL/neural network/tensorflow/process/nn_device.py @@ -2,13 +2,13 @@ import Note.nn.layer.dense as d # import Note's dense layer module from Note.nn.layer.flatten import flatten # import Note's flatten layer function from Note.nn.process.optimizer import Momentum # import Note's momentum optimizer module +from Note.nn.assign_device import assign_device_with_modulo # import the function to assign device according to the process index and the device type using modulo operation class nn: # A neural network class example, allocate device for multiple threads def __init__(self): # initialize the network self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) # loss object, kernel uses it to calculate loss. Here we use sparse categorical crossentropy with logits as output self.optimizer=Momentum(0.07,0.7) # optimizer, kernel uses it to optimize. Here we use a custom momentum optimizer with learning rate 0.07 and momentum 0.7 - self.device_table={0:'GPU:0',1:'GPU:0',2:'GPU:0',3:'GPU:1',4:'GPU:1',5:'GPU:1',6:'GPU:2'} # a dictionary to map process index to device name self.info='example' # some information about the network @@ -22,7 +22,7 @@ def build(self): # build function, kernel uses it to create the network layers def fp(self,data,p): # forward propagation function, kernel uses it for forward propagation - with tf.device(self.device_table[p]): # assign the device according to the process index p + with tf.device(assign_device_with_modulo(p,'GPU')): # assign the device according to the process index p data=flatten(data) # flatten the data to a vector of 784 elements output1=self.layer1.output(data) # pass the data through the first layer and get the output output2=self.layer2.output(output1) # pass the output of the first layer through the second layer and get the final output logits @@ -36,4 +36,4 @@ def loss(self,output,labels): # loss function, kernel uses it to calculate loss def opt(self,gradient): # optimization function, kernel uses it to optimize parameter # Perform optimization on the parameters using the gradient param=self.optimizer.opt(gradient,self.param) # apply the Note's momentum optimizer to update the parameters using the gradient - return param \ No newline at end of file + return param From 852544cfe0665fa8f979c2e001858dc981bb59de Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Wed, 19 Jul 2023 08:39:50 +0800 Subject: [PATCH 060/337] Update nn_device.py --- .../DL/neural network/tensorflow/process/nn_device.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/process/nn_device.py b/Note 7.0 documentation/DL/neural network/tensorflow/process/nn_device.py index e1720a74..15b33bf4 100644 --- a/Note 7.0 documentation/DL/neural network/tensorflow/process/nn_device.py +++ b/Note 7.0 documentation/DL/neural network/tensorflow/process/nn_device.py @@ -2,7 +2,7 @@ import Note.nn.layer.dense as d # import Note's dense layer module from Note.nn.layer.flatten import flatten # import Note's flatten layer function from Note.nn.process.optimizer import Momentum # import Note's momentum optimizer module -from Note.nn.assign_device import assign_device_with_modulo # import the function to assign device according to the process index and the device type using modulo operation +from Note.nn.process.assign_device import assign_device_with_modulo # import the function to assign device according to the process index and the device type using modulo operation class nn: # A neural network class example, allocate device for multiple threads From 3f9b138d742138854bf6d492d20eb59cb42e019d Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Wed, 19 Jul 2023 09:04:40 +0800 Subject: [PATCH 061/337] Update nn_device.py --- .../DL/neural network/tensorflow/process/nn_device.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/process/nn_device.py b/Note 7.0 documentation/DL/neural network/tensorflow/process/nn_device.py index 15b33bf4..9ca412c7 100644 --- a/Note 7.0 documentation/DL/neural network/tensorflow/process/nn_device.py +++ b/Note 7.0 documentation/DL/neural network/tensorflow/process/nn_device.py @@ -29,8 +29,9 @@ def fp(self,data,p): # forward propagation function, kernel uses it for forward return output2 - def loss(self,output,labels): # loss function, kernel uses it to calculate loss - return self.loss_object(labels,output) # return the mean softmax cross entropy loss between labels and output + def loss(self,output,labels,p): # loss function, kernel uses it to calculate loss + with tf.device(assign_device_with_modulo(p,'GPU')): # assign the device according to the process index p + return self.loss_object(labels,output) # return the mean softmax cross entropy loss between labels and output def opt(self,gradient): # optimization function, kernel uses it to optimize parameter From 9ced6a71ba174f6dad13634f2df7b5702a4d4948 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Wed, 19 Jul 2023 13:47:43 +0800 Subject: [PATCH 062/337] Update nn_device.py --- .../DL/neural network/tensorflow/process/nn_device.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/process/nn_device.py b/Note 7.0 documentation/DL/neural network/tensorflow/process/nn_device.py index 9ca412c7..1b37778c 100644 --- a/Note 7.0 documentation/DL/neural network/tensorflow/process/nn_device.py +++ b/Note 7.0 documentation/DL/neural network/tensorflow/process/nn_device.py @@ -2,7 +2,7 @@ import Note.nn.layer.dense as d # import Note's dense layer module from Note.nn.layer.flatten import flatten # import Note's flatten layer function from Note.nn.process.optimizer import Momentum # import Note's momentum optimizer module -from Note.nn.process.assign_device import assign_device_with_modulo # import the function to assign device according to the process index and the device type using modulo operation +from Note.nn.process.assign_device import assign_device # import the function to assign device according to the process index and the device type class nn: # A neural network class example, allocate device for multiple threads @@ -22,7 +22,7 @@ def build(self): # build function, kernel uses it to create the network layers def fp(self,data,p): # forward propagation function, kernel uses it for forward propagation - with tf.device(assign_device_with_modulo(p,'GPU')): # assign the device according to the process index p + with tf.device(assign_device(p,'GPU')): # assign the device according to the process index p data=flatten(data) # flatten the data to a vector of 784 elements output1=self.layer1.output(data) # pass the data through the first layer and get the output output2=self.layer2.output(output1) # pass the output of the first layer through the second layer and get the final output logits @@ -30,7 +30,7 @@ def fp(self,data,p): # forward propagation function, kernel uses it for forward def loss(self,output,labels,p): # loss function, kernel uses it to calculate loss - with tf.device(assign_device_with_modulo(p,'GPU')): # assign the device according to the process index p + with tf.device(assign_device(p,'GPU')): # assign the device according to the process index p return self.loss_object(labels,output) # return the mean softmax cross entropy loss between labels and output From 96e85c7c4f40b674d30202db024889f2e5e5cca4 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 20 Jul 2023 15:39:58 +0800 Subject: [PATCH 063/337] Update kernel.py --- .../DL/kernel/process/kernel.py | 42 +++++++++---------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/Note 7.0 documentation/DL/kernel/process/kernel.py b/Note 7.0 documentation/DL/kernel/process/kernel.py index 54629697..8f8f8486 100644 --- a/Note 7.0 documentation/DL/kernel/process/kernel.py +++ b/Note 7.0 documentation/DL/kernel/process/kernel.py @@ -46,18 +46,18 @@ def __init__(self,nn=None): self.total_epoch=0 # the total number of epochs def data(self,train_data=None,train_labels=None,test_data=None,test_labels=None,train_dataset=None,test_dataset=None): - if type(self.nn.param[0])!=list: + if train_data is not None and type(self.nn.param[0])!=list: self.train_data=train_data.astype(self.nn.param[0].dtype.name) # convert the train data type to match the model parameter type self.train_labels=train_labels.astype(self.nn.param[0].dtype.name) # convert the train labels type to match the model parameter type - else: + elif train_data is not None: self.train_data=train_data.astype(self.nn.param[0][0].dtype.name) # convert the train data type to match the model parameter type self.train_labels=train_labels.astype(self.nn.param[0][0].dtype.name) # convert the train labels type to match the model parameter type self.train_dataset=train_dataset # set the train dataset object - if test_data is not None: - self.test_data=test_data # set the test data array - self.test_labels=test_labels # set the test labels array - self.test_flag=True # set the test flag to True + self.test_data=test_data # set the test data array + self.test_labels=test_labels # set the test labels array self.test_dataset=test_dataset # set the test dataset object + if test_data is not None or test_dataset is not None: + self.test_flag=True # set the test flag to True self.batch_counter=np.zeros(self.process,dtype=np.int32) # initialize an array to count batches for each process if type(self.nn.param[0])!=list: # initialize an array to accumulate loss for each process self.total_loss=np.zeros(self.process,dtype=self.nn.param[0].dtype.name) @@ -73,17 +73,11 @@ def data(self,train_data=None,train_labels=None,test_data=None,test_labels=None, pass if self.priority_flag==True: self.opt_counter=np.zeros(self.process,dtype=np.int32) # initialize an array to count optimization steps for each process - if self.train_dataset==None: - if type(self.train_data)==list: - self.shape0=train_data[0].shape[0] # get the number of samples in the first train data array - self.batches=int((self.shape0-self.shape0%self.batch)/self.batch) # calculate the number of batches - if self.shape0%self.batch!=0: - self.batches+=1 # add one more batch if there are remaining samples - else: - self.shape0=train_data.shape[0] # get the number of samples in the train data array - self.batches=int((self.shape0-self.shape0%self.batch)/self.batch) # calculate the number of batches - if self.shape0%self.batch!=0: - self.batches+=1 # add one more batch if there are remaining samples + if train_data is not None: + self.shape0=train_data.shape[0] # get the number of samples in the train data array + self.batches=int((self.shape0-self.shape0%self.batch)/self.batch) # calculate the number of batches + if self.shape0%self.batch!=0: + self.batches+=1 # add one more batch if there are remaining samples if self.data_segment_flag==True: self.train_data,self.train_labels=self.segment_data() # segment the train data and labels according to the number of processes return @@ -478,17 +472,19 @@ def train(self,p,lock=None,g_lock=None,test_batch=None): def test(self,test_data=None,test_labels=None,batch=None,p=None): - if type(self.nn.param[0])!=list: + if test_data is not None and type(self.nn.param[0])!=list: test_data=test_data.astype(self.nn.param[0].dtype.name) # convert the test data type to match the model parameter type test_labels=test_labels.astype(self.nn.param[0].dtype.name) # convert the test labels type to match the model parameter type - else: + elif test_data is not None: test_data=test_data.astype(self.nn.param[0][0].dtype.name) # convert the test data type to match the model parameter type test_labels=test_labels.astype(self.nn.param[0][0].dtype.name) # convert the test labels type to match the model parameter type if batch!=None: # check if batch is not None total_loss=0 # initialize total loss to zero total_acc=0 # initialize total accuracy to zero if self.test_dataset!=None: # check if test dataset is not None + batches=0 for data_batch,labels_batch in self.test_dataset: # loop over the test dataset batches + batches+=1 try: try: output=self.nn.fp(data_batch) # use the neural network's forward propagation function to get the output @@ -507,9 +503,13 @@ def test(self,test_data=None,test_labels=None,batch=None,p=None): raise e except Exception: pass + test_loss=total_loss.numpy()/batches # calculate the average test loss for all batches + try: + if self.nn.accuracy!=None: # check if the neural network has an accuracy function + test_acc=total_acc.numpy()/batches # calculate the average test accuracy for all batches + except Exception: + pass else: # if test dataset is None - total_loss=0 # initialize total loss to zero - total_acc=0 # initialize total accuracy to zero batches=int((test_data.shape[0]-test_data.shape[0]%batch)/batch) # calculate the number of batches shape0=test_data.shape[0] # get the number of samples in the test data array for j in range(batches): # loop over the batches From fe4a5bf9e78b39ccb232b315ad6459d46203374f Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 20 Jul 2023 15:40:11 +0800 Subject: [PATCH 064/337] Update kernel.py --- Note 7.0 documentation/DL/kernel/kernel.py | 28 ++++++++++++---------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/Note 7.0 documentation/DL/kernel/kernel.py b/Note 7.0 documentation/DL/kernel/kernel.py index 1ff522be..ec624d93 100644 --- a/Note 7.0 documentation/DL/kernel/kernel.py +++ b/Note 7.0 documentation/DL/kernel/kernel.py @@ -41,23 +41,19 @@ def __init__(self,nn=None): def data(self,train_data=None,train_labels=None,test_data=None,test_labels=None,train_dataset=None,test_dataset=None): - if type(self.nn.param[0])!=list: + if train_data is not None and type(self.nn.param[0])!=list: self.train_data=train_data.astype(self.nn.param[0].dtype.name) # convert the train data type to match the neural network parameters type self.train_labels=train_labels.astype(self.nn.param[0].dtype.name) # convert the train labels type to match the neural network parameters type - else: + elif train_data is not None: self.train_data=train_data.astype(self.nn.param[0][0].dtype.name) # convert the train data type to match the neural network parameters type (for multiple inputs) self.train_labels=train_labels.astype(self.nn.param[0][0].dtype.name) # convert the train labels type to match the neural network parameters type (for multiple inputs) self.train_dataset=train_dataset # a tensorflow or pytorch dataset object for train data (optional) - if test_data is not None: - if type(self.nn.param[0])!=list: - self.test_data=test_data.astype(self.nn.param[0].dtype.name) # convert the test data type to match the neural network parameters type - self.test_labels=test_labels.astype(self.nn.param[0].dtype.name) # convert the test labels type to match the neural network parameters type - else: - self.test_data=test_data.astype(self.nn.param[0][0].dtype.name) # convert the test data type to match the neural network parameters type (for multiple inputs) - self.test_labels=test_labels.astype(self.nn.param[0][0].dtype.name) # convert the test labels type to match the neural network parameters type (for multiple inputs) - self.test_flag=True # set the test flag to True if test data is provided + self.test_data=test_data # set the test data array + self.test_labels=test_labels # set the test labels array self.test_dataset=test_dataset # a tensorflow or pytorch dataset object for test data (optional) - if self.train_dataset==None: + if test_data is not None or test_dataset is not None: + self.test_flag=True # set the test flag to True if test data is provided + if train_data is not None: self.shape0=train_data.shape[0] # get the number of samples in train data return @@ -546,7 +542,9 @@ def test(self,test_data=None,test_labels=None,batch=None): # a method to calcul total_loss=0 # initialize the total loss value for all batches total_acc=0 # initialize the total accuracy value for all batches if self.test_dataset!=None: # if a tensorflow or pytorch dataset object is used for test data + batches=0 for data_batch,labels_batch in self.test_dataset: # iterate over each batch of data and labels from the test dataset + batches+=1 output=self.nn.fp(data_batch) # get the output value using the neural network's forward propagation method batch_loss=self.nn.loss(output,labels_batch) # get the batch loss value using the neural network's loss function total_loss+=batch_loss # accumulate the batch loss to total loss @@ -559,9 +557,13 @@ def test(self,test_data=None,test_labels=None,batch=None): # a method to calcul raise e # raise the exception except Exception: pass + test_loss=total_loss.numpy()/batches # convert the total loss value to numpy array and divide it by number of batches to get the average loss value for test data + try: + if self.nn.accuracy!=None: # if the neural network has an accuracy method defined + test_acc=total_acc.numpy()/batches # convert the total accuracy value to numpy array and divide it by number of batches to get the average accuracy value for test data + except Exception: + pass else: # if a numpy array is used for test data - total_loss=0 # initialize the total loss value for all batches - total_acc=0 # initialize the total accuracy value for all batches batches=int((test_data.shape[0]-test_data.shape[0]%batch)/batch) # calculate how many batches are needed for test data according to batch size shape0=test_data.shape[0] # get the number of samples in test data for j in range(batches): # iterate over each batch index From 44efef75516afc817e1548e1fa39069a656880cd Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 20 Jul 2023 19:11:32 +0800 Subject: [PATCH 065/337] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7e1f3085..e51f4277 100644 --- a/README.md +++ b/README.md @@ -264,7 +264,7 @@ for p in range(7): #loop over the processes import Note.DL.process.kernel as k #import kernel module import tensorflow as tf #import tensorflow library import nn as n #import neural network module -from multiprocessing import Process,Lock,Manager #import multiprocessing tools +from multiprocessing import Process,Manager #import multiprocessing tools mnist=tf.keras.datasets.mnist #load mnist dataset (x_train,y_train),(x_test,y_test)=mnist.load_data() #split data into train and test sets x_train,x_test =x_train/255.0,x_test/255.0 #normalize data From 589708aaa0821eb7993ff4e62f39242fd928cefe Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 20 Jul 2023 20:59:54 +0800 Subject: [PATCH 066/337] Update kernel.py --- Note 7.0 documentation/DL/kernel/process/kernel.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Note 7.0 documentation/DL/kernel/process/kernel.py b/Note 7.0 documentation/DL/kernel/process/kernel.py index 8f8f8486..f435eaf1 100644 --- a/Note 7.0 documentation/DL/kernel/process/kernel.py +++ b/Note 7.0 documentation/DL/kernel/process/kernel.py @@ -128,18 +128,17 @@ def init(self,manager): if self.priority_flag==True: self.opt_counter=Array('i',self.opt_counter) # create a shared array for optimization counter try: - if self.nn.attenuate!=None: - self.nn.opt_counter=manager.list([self.nn.opt_counter]) # create a shared list for the neural network's optimization counter + self.nn.opt_counter=manager.list([self.nn.opt_counter]) # create a shared list for the neural network's optimization counter except Exception: - pass + self.opt_counter_=manager.list() try: self.nn.ec=manager.list([self.nn.ec]) # create a shared list for the neural network's epoch counter except Exception: - pass + self.ec_=manager.list() try: self.nn.bc=manager.list([self.nn.bc]) # create a shared list for the neural network's batch counter except Exception: - pass + self.bc_=manager.list() self.stop_flag=Value('b',self.stop_flag) # create a shared value for stop flag self.save_flag=Value('b',self.save_flag) # create a shared value for save flag self.param=manager.dict() # create a shared dictionary for parameters From 174bf333b3ca9f6b8ee2c4f3debd2b7c06e89a4b Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 20 Jul 2023 21:00:12 +0800 Subject: [PATCH 067/337] Update kernel.py --- Note 7.0 documentation/RL/kernel/kernel.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Note 7.0 documentation/RL/kernel/kernel.py b/Note 7.0 documentation/RL/kernel/kernel.py index 0c7a3c74..abbdc720 100644 --- a/Note 7.0 documentation/RL/kernel/kernel.py +++ b/Note 7.0 documentation/RL/kernel/kernel.py @@ -68,18 +68,17 @@ def init(self,manager): if self.priority_flag==True: # if priority optimization is enabled self.opt_counter=Array('i',np.zeros(self.process,dtype=np.int32)) # use Array to store optimization counter for each process try: - if self.nn.attenuate!=None: # if attenuation function is defined - self.nn.opt_counter=manager.list([self.nn.opt_counter]) # use manager.list to share optimization counter for attenuation function among processes + self.nn.opt_counter=manager.list([self.nn.opt_counter]) # use manager.list to share optimization counter for attenuation function among processes except Exception: - pass + self.opt_counter_=manager.list() try: self.nn.ec=manager.list(self.nn.ec) # use manager.list to share episode counter for neural network model among processes except Exception: - pass + self.ec_=manager.list() try: self.nn.bc=manager.list(self.nn.bc) # use manager.list to share batch counter for neural network model among processes except Exception: - pass + self.bc_=manager.list() self.stop_flag=Value('b',self.stop_flag) # use Value to share stop flag among processes self.save_flag=Value('b',self.save_flag) # use Value to share save flag among processes self.param=manager.dict() # use manager.dict to share parameters among processes From 7a2b1f56a73e7e84e70dcfe27d58895f78c6fc38 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Fri, 21 Jul 2023 19:31:23 +0800 Subject: [PATCH 068/337] Add files via upload --- .../DL/neural network/pytorch/process/nn.py | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 Note 7.0 documentation/DL/neural network/pytorch/process/nn.py diff --git a/Note 7.0 documentation/DL/neural network/pytorch/process/nn.py b/Note 7.0 documentation/DL/neural network/pytorch/process/nn.py new file mode 100644 index 00000000..00b0ed4b --- /dev/null +++ b/Note 7.0 documentation/DL/neural network/pytorch/process/nn.py @@ -0,0 +1,42 @@ +import torch +from torch import nn + + +class NeuralNetwork(nn.Module): + def __init__(self): + super().__init__() + self.flatten = nn.Flatten() + self.linear_relu_stack=nn.Sequential( + nn.Linear(28*28,512), + nn.ReLU(), + nn.Linear(512,512), + nn.ReLU(), + nn.Linear(512,10) + ) + + + def forward(self,x): + x = self.flatten(x) + logits=self.linear_relu_stack(x) + return logits + + +class neuralnetwork: + def __init__(self): + if torch.cuda.is_available(): + self.device=torch.device('cuda') + else: + self.device=torch.device('cpu') + self.model=NeuralNetwork().to(self.device) + self.loss_fn=nn.CrossEntropyLoss() + self.opt=[torch.optim.SGD(self.model.parameters(),lr=1e-3) for _ in range(3)] + + + def fp(self,x): + pred=self.model(x.to(self.device)) + return pred + + + def loss(self,output,labels): + loss=self.loss_fn(output,labels.to(self.device)) + return loss \ No newline at end of file From 563520b506d1cbda562187703af0fdcb47630783 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Sat, 22 Jul 2023 14:01:16 +0800 Subject: [PATCH 069/337] Update nn.py --- .../DL/neural network/pytorch/process/nn.py | 61 +++++++++---------- 1 file changed, 30 insertions(+), 31 deletions(-) diff --git a/Note 7.0 documentation/DL/neural network/pytorch/process/nn.py b/Note 7.0 documentation/DL/neural network/pytorch/process/nn.py index 00b0ed4b..7347d11d 100644 --- a/Note 7.0 documentation/DL/neural network/pytorch/process/nn.py +++ b/Note 7.0 documentation/DL/neural network/pytorch/process/nn.py @@ -1,42 +1,41 @@ -import torch -from torch import nn +import torch # import the PyTorch library +from torch import nn # import the neural network module from PyTorch - -class NeuralNetwork(nn.Module): - def __init__(self): - super().__init__() - self.flatten = nn.Flatten() - self.linear_relu_stack=nn.Sequential( - nn.Linear(28*28,512), - nn.ReLU(), - nn.Linear(512,512), - nn.ReLU(), - nn.Linear(512,10) +class NeuralNetwork(nn.Module): # define a class for the neural network model + def __init__(self): # define the constructor method + super().__init__() # call the parent class constructor + self.flatten = nn.Flatten() # define a layer to flatten the input + self.linear_relu_stack=nn.Sequential( # define a stack of linear and relu layers + nn.Linear(28*28,512), # a linear layer with 28*28 input features and 512 output features + nn.ReLU(), # a relu activation function + nn.Linear(512,512), # another linear layer with 512 input and output features + nn.ReLU(), # another relu activation function + nn.Linear(512,10) # a final linear layer with 512 input features and 10 output features ) - def forward(self,x): - x = self.flatten(x) - logits=self.linear_relu_stack(x) - return logits + def forward(self,x): # define the forward method + x = self.flatten(x) # flatten the input x + logits=self.linear_relu_stack(x) # pass x through the linear relu stack and get the logits + return logits # return the logits -class neuralnetwork: - def __init__(self): - if torch.cuda.is_available(): - self.device=torch.device('cuda') +class neuralnetwork: # define another class for the neural network object + def __init__(self): # define the constructor method + if torch.cuda.is_available(): # check if cuda is available + self.device=torch.device('cuda') # set the device to cuda else: - self.device=torch.device('cpu') - self.model=NeuralNetwork().to(self.device) - self.loss_fn=nn.CrossEntropyLoss() - self.opt=[torch.optim.SGD(self.model.parameters(),lr=1e-3) for _ in range(3)] + self.device=torch.device('cpu') # otherwise set the device to cpu + self.model=NeuralNetwork().to(self.device) # create an instance of the NeuralNetwork class and move it to the device + self.loss_fn=nn.CrossEntropyLoss() # define the loss function as cross entropy loss + self.opt=[torch.optim.SGD(self.model.parameters(),lr=1e-3) for _ in range(3)] # define a list of three stochastic gradient descent optimizers with learning rate 1e-3 - def fp(self,x): - pred=self.model(x.to(self.device)) - return pred + def fp(self,x): # define a method for forward propagation + pred=self.model(x.to(self.device)) # get the predictions from the model by passing x to the device + return pred # return the predictions - def loss(self,output,labels): - loss=self.loss_fn(output,labels.to(self.device)) - return loss \ No newline at end of file + def loss(self,output,labels): # define a method for calculating the loss + loss=self.loss_fn(output,labels.to(self.device)) # compute the loss by passing output and labels to the device + return loss # return the loss From ab7fc9cc5f311c6315045dd13563bcf6d143612b Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Sat, 22 Jul 2023 14:01:34 +0800 Subject: [PATCH 070/337] Add files via upload --- .../pytorch/process/nn_device.py | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 Note 7.0 documentation/DL/neural network/pytorch/process/nn_device.py diff --git a/Note 7.0 documentation/DL/neural network/pytorch/process/nn_device.py b/Note 7.0 documentation/DL/neural network/pytorch/process/nn_device.py new file mode 100644 index 00000000..4c890f52 --- /dev/null +++ b/Note 7.0 documentation/DL/neural network/pytorch/process/nn_device.py @@ -0,0 +1,45 @@ +import torch # import the PyTorch library +from torch import nn # import the neural network module from PyTorch +from Note.nn.assign_device import assign_device_pytorch # import a custom function to assign device for PyTorch tensors + + +class NeuralNetwork(nn.Module): # define a class for the neural network model + def __init__(self): # define the constructor method + super().__init__() # call the parent class constructor + self.flatten = nn.Flatten() # define a layer to flatten the input tensor + self.linear_relu_stack=nn.Sequential( # define a stack of linear and relu layers + nn.Linear(28*28,512), # define a linear layer with input size 28*28 and output size 512 + nn.ReLU(), # define a relu activation function + nn.Linear(512,512), # define another linear layer with input and output size 512 + nn.ReLU(), # define another relu activation function + nn.Linear(512,10) # define the final linear layer with output size 10 + ) + + + def forward(self,x): # define the forward method for the model + x = self.flatten(x) # flatten the input tensor + logits=self.linear_relu_stack(x) # pass the flattened tensor through the linear relu stack + return logits # return the logits as the output + + +class neuralnetwork: # define another class for the neural network object + def __init__(self): # define the constructor method + if torch.cuda.is_available(): # check if cuda device is available + self.device=torch.device('cuda') # set the device to cuda + else: # otherwise + self.device=torch.device('cpu') # set the device to cpu + self.model=NeuralNetwork().to(self.device) # create an instance of the neural network model and move it to the device + self.loss_fn=nn.CrossEntropyLoss() # define the loss function as cross entropy loss + self.opt=[torch.optim.SGD(self.model.parameters(),lr=1e-3) for _ in range(3)] # define a list of optimizers as stochastic gradient descent with learning rate 1e-3 + + + def fp(self,x,p): # define a method for forward propagation + pred=self.model(x.to(assign_device_pytorch(p,'GPU'))) # assign the device according to the process index p + # get the predictions from the model by passing the input tensor to the device and then to the model + return pred # return the predictions + + + def loss(self,output,labels,p): # define a method for calculating loss + loss=self.loss_fn(output,labels.to(assign_device_pytorch(p,'GPU'))) # assign the device according to the process index p + # calculate the loss by passing the output and labels tensors to the device and then to the loss function + return loss # return the loss \ No newline at end of file From 42b60f944551563892047ec59e6813c9efd6bb50 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Sat, 22 Jul 2023 19:59:13 +0800 Subject: [PATCH 071/337] Update kernel.py --- Note 7.0 documentation/DL/kernel/kernel.py | 1444 ++++++++++---------- 1 file changed, 732 insertions(+), 712 deletions(-) diff --git a/Note 7.0 documentation/DL/kernel/kernel.py b/Note 7.0 documentation/DL/kernel/kernel.py index ec624d93..92ecad36 100644 --- a/Note 7.0 documentation/DL/kernel/kernel.py +++ b/Note 7.0 documentation/DL/kernel/kernel.py @@ -1,715 +1,735 @@ -from tensorflow import function -import numpy as np -import matplotlib.pyplot as plt -import time +from tensorflow import function # import the function decorator from tensorflow +from multiprocessing import Process # import the Process class from multiprocessing +import numpy as np # import numpy as np +from Note.DL.dl.test import parallel_test # import the parallel_test class from Note.DL.dl.test +import time # import time -class kernel: - def __init__(self,nn=None): - self.nn=nn # the neural network object - try: - self.nn.km=1 # a flag to indicate the kernel mode - except Exception: - pass - self.platform=None # the platform to use, either tensorflow or pytorch - self.batches=None # the number of batches for training data - self.suspend=False # a flag to indicate whether to suspend the training - self.stop=False # a flag to indicate whether to stop the training - self.stop_flag=False # a flag to indicate whether the training has been stopped - self.save_epoch=None # the epoch number to save the model - self.batch=None # the batch size for training data - self.epoch=0 # the current epoch number - self.end_loss=None # the target loss value to end the training - self.end_acc=None # the target accuracy value to end the training - self.end_test_loss=None # the target test loss value to end the training - self.end_test_acc=None # the target test accuracy value to end the training - self.acc_flag='%' # a flag to indicate whether to use percentage or decimal for accuracy display - self.train_counter=0 # a counter for how many times the train method has been called - self.filename='save.dat' # the file name to save the model - self.train_loss=None # the current train loss value - self.train_acc=None # the current train accuracy value - self.train_loss_list=[] # a list of train loss values for each epoch - self.train_acc_list=[] # a list of train accuracy values for each epoch - self.test_loss=None # the current test loss value - self.test_acc=None # the current test accuracy value - self.test_loss_list=[] # a list of test loss values for each epoch - self.test_acc_list=[] # a list of test accuracy values for each epoch - self.test_flag=False # a flag to indicate whether to use test data or not - self.total_epoch=0 # the total number of epochs for all trainings - self.time=0 # the time elapsed for one training session - self.total_time=0 # the total time elapsed for all trainings - - - def data(self,train_data=None,train_labels=None,test_data=None,test_labels=None,train_dataset=None,test_dataset=None): - if train_data is not None and type(self.nn.param[0])!=list: - self.train_data=train_data.astype(self.nn.param[0].dtype.name) # convert the train data type to match the neural network parameters type - self.train_labels=train_labels.astype(self.nn.param[0].dtype.name) # convert the train labels type to match the neural network parameters type - elif train_data is not None: - self.train_data=train_data.astype(self.nn.param[0][0].dtype.name) # convert the train data type to match the neural network parameters type (for multiple inputs) - self.train_labels=train_labels.astype(self.nn.param[0][0].dtype.name) # convert the train labels type to match the neural network parameters type (for multiple inputs) - self.train_dataset=train_dataset # a tensorflow or pytorch dataset object for train data (optional) - self.test_data=test_data # set the test data array - self.test_labels=test_labels # set the test labels array - self.test_dataset=test_dataset # a tensorflow or pytorch dataset object for test data (optional) - if test_data is not None or test_dataset is not None: - self.test_flag=True # set the test flag to True if test data is provided - if train_data is not None: - self.shape0=train_data.shape[0] # get the number of samples in train data - return - - - def init(self): # a method to initialize the attributes for a new training session - self.suspend=False - self.stop=False - self.stop_flag=False - self.save_epoch=None - self.end_loss=None - self.end_acc=None - self.end_test_loss=None - self.end_test_acc=None - self.train_loss=None - self.train_acc=None - self.test_loss=None - self.test_acc=None - self.train_loss_list.clear() - self.train_acc_list.clear() - self.test_loss_list.clear() - self.test_acc_list.clear() - self.test_flag=False - self.train_counter=0 - self.epoch=0 - self.total_epoch=0 - self.time=0 - self.total_time=0 - return - - - def end(self): # a method to check whether the training has reached the target loss or accuracy values - if self.end_loss!=None and len(self.train_loss_list)!=0 and self.train_loss_list[-1]self.end_acc: # if the target train accuracy is given and the current train accuracy is higher than it - return True - elif self.end_loss!=None and len(self.train_loss_list)!=0 and self.end_acc!=None and self.train_loss_list[-1]self.end_acc: # if both the target train loss and accuracy are given and the current train loss is lower than it and the current train accuracy is higher than it - return True - elif self.end_test_loss!=None and len(self.test_loss_list)!=0 and self.test_loss_list[-1]self.end_test_acc: # if the target test accuracy is given and the current test accuracy is higher than it - return True - elif self.end_test_loss!=None and len(self.test_loss_list)!=0 and self.end_test_acc!=None and self.test_loss_list[-1]self.end_test_acc: # if both the target test loss and accuracy are given and the current test loss is lower than it and the current test accuracy is higher than it - return True - - - def loss_acc(self,output=None,labels_batch=None,loss=None,test_batch=None,total_loss=None,total_acc=None): # a method to calculate the loss and accuracy values for each batch or epoch - if self.batch!=None: # if batch mode is used - total_loss+=loss # accumulate the batch loss to total loss - try: - batch_acc=self.nn.accuracy(output,labels_batch) # calculate the batch accuracy using the neural network's accuracy method - total_acc+=batch_acc # accumulate the batch accuracy to total accuracy - except Exception as e: - try: - if self.nn.accuracy!=None: # if the neural network has an accuracy method defined - raise e # raise the exception - except Exception: - pass - return total_loss,total_acc # return the total loss and accuracy values for all batches so far - else: # if batch mode is not used (use all data at once) - loss=loss.numpy() # convert the loss value to numpy array - self.train_loss=loss # assign the loss value to train loss attribute - self.train_loss_list.append(loss) # append the loss value to train loss list - try: - acc=self.nn.accuracy(output,self.train_labels) # calculate the accuracy value using the neural network's accuracy method - acc=acc.numpy() # convert the accuracy value to numpy array - self.train_acc=acc # assign the accuracy value to train accuracy attribute - self.train_acc_list.append(acc) # append the accuracy value to train accuracy list - except Exception as e: - try: - if self.nn.accuracy!=None: # if the neural network has an accuracy method defined - raise e # raise the exception - except Exception: - pass - if self.test_flag==True: # if test data is used - self.test_loss,self.test_acc=self.test(self.test_data,self.test_labels,test_batch) # calculate the test loss and accuracy values using the test method - self.test_loss_list.append(self.test_loss) # append the test loss value to test loss list - try: - self.test_acc_list.append(self.test_acc) # append the test accuracy value to test accuracy list - except Exception as e: - try: - if self.nn.accuracy!=None: # if the neural network has an accuracy method defined - raise e # raise the exception - except Exception: - pass - return # return nothing for this case - - - def data_func(self,data_batch=None,labels_batch=None,batch=None,index1=None,index2=None,j=None,flag=None): # a method to get a batch of data and labels from the train data and labels - if flag==None: # if flag is None, it means the batch size is smaller than the number of samples - if batch!=1: # if batch size is not 1 - data_batch=self.train_data[index1:index2] # get a slice of train data according to the index range - else: # if batch size is 1 - data_batch=self.train_data[j] # get one sample of train data according to the index - if batch!=1: # if batch size is not 1 - labels_batch=self.train_labels[index1:index2] # get a slice of train labels according to the index range - else: # if batch size is 1 - labels_batch=self.train_labels[j] # get one sample of train labels according to the index - else: # if flag is not None, it means the batch size is larger than the number of samples - try: - try: - data_batch=self.platform.concat([self.train_data[index1:],self.train_data[:index2]],0) # concatenate two slices of train data from the end and the beginning to form a batch - labels_batch=self.platform.concat([self.train_labels[index1:],self.train_labels[:index2]],0) # concatenate two slices of train labels from the end and the beginning to form a batch - except Exception: # if the platform's concat method fails - data_batch=np.concatenate([self.train_data[index1:],self.train_data[:index2]],0) # use numpy's concatenate method instead - labels_batch=np.concatenate([self.train_labels[index1:],self.train_labels[:index2]],0) # use numpy's concatenate method instead - except Exception as e: - raise e # raise any other exception - return data_batch,labels_batch # return the batch of data and labels - - - @function(jit_compile=True) # use tensorflow's function decorator to speed up the execution - def tf_opt(self,data,labels): # a method to perform one optimization step using tensorflow platform - try: - try: - if self.nn.GradientTape!=None: # if the neural network has a GradientTape method defined - tape,output,loss=self.nn.GradientTape(data,labels) # use the neural network's GradientTape method to get the tape, output and loss values - except Exception: # if the neural network does not have a GradientTape method defined or it fails - with self.platform.GradientTape(persistent=True) as tape: # use tensorflow's GradientTape context manager instead - try: - output=self.nn.fp(data) # get the output value using the neural network's forward propagation method - loss=self.nn.loss(output,labels) # get the loss value using the neural network's loss function - except Exception: # if the neural network's forward propagation method or loss function fails or they are combined in one method - output,loss=self.nn.fp(data,labels) # use the neural network's forward propagation method with both data and labels as inputs to get the output and loss values - except Exception as e: - raise e # raise any other exception - try: - try: - gradient=self.nn.gradient(tape,loss) # use the neural network's gradient method to get the gradient value from the tape and loss values - except Exception: # if the neural network does not have a gradient method defined or it fails - gradient=tape.gradient(loss,self.nn.param) # use tensorflow's tape.gradient method instead with loss value and neural network parameters as inputs - except Exception as e: - raise e # raise any other exception - try: - try: - self.nn.opt.apply_gradients(zip(gradient,self.nn.param)) # use the neural network's optimizer's apply_gradients method to update the neural network parameters with gradient value - except Exception: # if the neural network does not have an optimizer or its apply_gradients method fails - self.nn.opt(gradient) # use the neural network's optimizer directly with gradient value as input - except Exception as e: - raise e # raise any other exception - return output,loss # return the output and loss values for this optimization step - - - def pytorch_opt(self,data,labels): # a method to perform one optimization step using pytorch platform - output=self.nn.fp(data) # get the output value using the neural network's forward propagation method - loss=self.nn.loss(output,labels) # get the loss value using the neural network's loss function - try: - try: - self.nn.opt.zero_grad() # use the neural network's optimizer's zero_grad method to clear the previous gradients - loss.backward() # use pytorch's loss.backward method to calculate the gradients - self.nn.opt.step() # use the neural network's optimizer's step method to update the neural network parameters with gradient value - except Exception: # if the neural network does not have an optimizer or its zero_grad or step method fails - self.nn.opt(loss) # use the neural network's optimizer directly with loss value as input - except Exception as e: - raise e # raise any other exception - return output,loss # return the output and loss values for this optimization step - - - def opt(self,data,labels): # a method to perform one optimization step using either tensorflow or pytorch platform - try: - try: - if self.platform.DType!=None: # if tensorflow platform is used - output,loss=self.tf_opt(data,labels) # use the tf_opt method for optimization - except Exception: # if tensorflow platform is not used or it fails - output,loss=self.pytorch_opt(data,labels) # use the pytorch_opt method for optimization - except Exception as e: - raise e # raise any other exception - return output,loss # return the output and loss values for this optimization step - - - def _train(self,batch=None,test_batch=None): # a method to perform one epoch of training - if batch!=None: # if batch mode is used - total_loss=0 # initialize the total loss value for all batches - total_acc=0 # initialize the total accuracy value for all batches - if self.train_dataset!=None: # if a tensorflow or pytorch dataset object is used for train data - for data_batch,labels_batch in self.train_dataset: # iterate over each batch of data and labels from the train dataset - if self.stop==True: # if the stop flag is set to True - if self.stop_func(): # check whether the training has reached the target loss or accuracy values using the stop_func method - return # return nothing and end the training - try: - data_batch,labels_batch=self.nn.data_func(data_batch,labels_batch) # use the neural network's data_func method to preprocess the data and labels batch (optional) - except Exception as e: - try: - if self.nn.data_func!=None: # if the neural network has a data_func method defined - raise e # raise the exception - except Exception: - pass - output,batch_loss=self.opt(data_batch,labels_batch) # perform one optimization step using the opt method and get the output and batch loss values - total_loss,total_acc=self.loss_acc(output=output,labels_batch=labels_batch,loss=batch_loss,total_loss=total_loss,total_acc=total_acc) # calculate and update the total loss and accuracy values for all batches using the loss_acc method - else: # if a numpy array is used for train data - total_loss=0 # initialize the total loss value for all batches - total_acc=0 # initialize the total accuracy value for all batches - batches=int((self.shape0-self.shape0%batch)/batch) # calculate how many batches are needed for train data according to batch size - for j in range(batches): # iterate over each batch index - if self.stop==True: # if the stop flag is set to True - if self.stop_func(): # check whether the training has reached the target loss or accuracy values using the stop_func method - return # return nothing and end the training - index1=j*batch # calculate the start index of train data for this batch - index2=(j+1)*batch # calculate the end index of train data for this batch - data_batch,labels_batch=self.data_func(batch,index1,index2,j) # get a batch of data and labels from train data and labels using the data_func method - try: - data_batch,labels_batch=self.nn.data_func(data_batch,labels_batch) # use the neural network's data_func method to preprocess the data and labels batch (optional) - except Exception as e: - try: - if self.nn.data_func!=None: # if the neural network has a data_func method defined - raise e # raise the exception - except Exception: - pass - output,batch_loss=self.opt(data_batch,labels_batch) # perform one optimization step - total_loss,total_acc=self.loss_acc(output=output,labels_batch=labels_batch,loss=batch_loss,total_loss=total_loss,total_acc=total_acc) # calculate and update the total loss and accuracy values for all batches using the loss_acc method - try: - try: - self.nn.bc.assign_add(1) # use the neural network's batch counter's assign_add method to increase the batch counter by 1 (for tensorflow platform) - except Exception: # if the neural network does not have a batch counter or its assign_add method fails - self.nn.bc+=1 # use the normal addition operation to increase the batch counter by 1 (for pytorch platform) - except Exception: - pass - if self.shape0%batch!=0: # if there are some samples left in train data that are not enough to form a full batch - if self.stop==True: # if the stop flag is set to True - if self.stop_func(): # check whether the training has reached the target loss or accuracy values using the stop_func method - return # return nothing and end the training - batches+=1 # increase the number of batches by 1 to include the remaining samples - index1=batches*batch # calculate the start index of train data for the last batch - index2=batch-(self.shape0-batches*batch) # calculate how many samples are needed from the beginning of train data to form a full batch - data_batch,labels_batch=self.data_func(batch,index1,index2,flag=True) # get a batch of data and labels from train data and labels using the data_func method with flag set to True - try: - data_batch,labels_batch=self.nn.data_func(data_batch,labels_batch) # use the neural network's data_func method to preprocess the data and labels batch (optional) - except Exception as e: - try: - if self.nn.data_func!=None: # if the neural network has a data_func method defined - raise e # raise the exception - except Exception: - pass - output,batch_loss=self.opt(data_batch,labels_batch) # perform one optimization step using the opt method and get the output and batch loss values - total_loss,total_acc=self.loss_acc(output=output,labels_batch=labels_batch,loss=batch_loss,total_loss=total_loss,total_acc=total_acc) # calculate and update the total loss and accuracy values for all batches using the loss_acc method - try: - try: - self.nn.bc.assign_add(1) # use the neural network's batch counter's assign_add method to increase the batch counter by 1 (for tensorflow platform) - except Exception: # if the neural network does not have a batch counter or its assign_add method fails - self.nn.bc+=1 # use the normal addition operation to increase the batch counter by 1 (for pytorch platform) - except Exception: - pass - try: - if self.platform.DType!=None: # if tensorflow platform is used - loss=total_loss.numpy()/batches # convert the total loss value to numpy array and divide it by number of batches to get the average loss value for this epoch - except Exception: # if tensorflow platform is not used or it fails - loss=total_loss.detach().numpy()/batches # detach the total loss value from computation graph and convert it to numpy array and divide it by number of batches to get the average loss value for this epoch - try: - train_acc=total_acc.numpy()/batches # convert the total accuracy value to numpy array and divide it by number of batches to get the average accuracy value for this epoch - except Exception as e: - try: - if self.nn.accuracy!=None: # if the neural network has an accuracy method defined - raise e # raise the exception - except Exception: - pass - self.train_loss=loss # assign the average loss value to train loss attribute - self.train_loss_list.append(loss) # append the average loss value to train loss list - try: - self.train_acc=train_acc # assign the average accuracy value to train accuracy attribute - self.train_acc_list.append(train_acc) # append the average accuracy value to train accuracy list +class kernel: # define a class + def __init__(self,nn=None): + # define the constructor + self.nn=nn # assign the nn argument to the self.nn attribute + if hasattr(self.nn,'km'): # check if the nn object has the km attribute + self.nn.km=1 # set the km attribute to 1 + self.platform=None # initialize the platform attribute to None + self.batches=None # initialize the batches attribute to None + self.process_t=None # initialize the process_t attribute to None + self.prefetch_batch_size_t=None # initialize the prefetch_batch_size_t attribute to None + self.suspend=False # initialize the suspend attribute to False + self.stop=False # initialize the stop attribute to False + self.stop_flag=False # initialize the stop_flag attribute to False + self.save_epoch=None # initialize the save_epoch attribute to None + self.batch=None # initialize the batch attribute to None + self.epoch=0 # initialize the epoch attribute to 0 + self.end_loss=None # initialize the end_loss attribute to None + self.end_acc=None # initialize the end_acc attribute to None + self.end_test_loss=None # initialize the end_test_loss attribute to None + self.end_test_acc=None # initialize the end_test_acc attribute to None + self.acc_flag='%' # initialize the acc_flag attribute to '%' + self.train_counter=0 # initialize the train_counter attribute to 0 + self.filename='save.dat' # initialize the filename attribute to 'save.dat' + self.train_loss=None # initialize the train_loss attribute to None + self.train_acc=None # initialize the train_acc attribute to None + self.train_loss_list=[] # initialize the train_loss_list attribute to an empty list + self.train_acc_list=[] # initialize the train_acc_list attribute to an empty list + self.test_loss=None # initialize the test_loss attribute to None + self.test_acc=None # initialize the test_acc attribute to None + self.test_loss_list=[] # initialize the test_loss_list attribute to an empty list + self.test_acc_list=[] # initialize the test_acc_list attribute to an empty list + self.test_flag=False # initialize the test_flag attribute to False + self.total_epoch=0 # initialize the total_epoch attribute to 0 + self.time=0 # initialize the time attribute to 0 + self.total_time=0 # initialize the total_time attribute to 0 + + + def data(self,train_data=None,train_labels=None,test_data=None,test_labels=None,train_dataset=None,test_dataset=None): + # define a method for setting up data attributes + if train_data is not None and type(self.nn.param[0])!=list: # check if train_data is not None and the type of the first element of self.nn.param is not list + self.train_data=train_data.astype(self.nn.param[0].dtype.name) # convert the train_data to the same data type as the first element of self.nn.param and assign it to self.train_data + self.train_labels=train_labels.astype(self.nn.param[0].dtype.name) # convert the train_labels to the same data type as the first element of self.nn.param and assign it to self.train_labels + elif train_data is not None: # check if train_data is not None + self.train_data=train_data.astype(self.nn.param[0][0].dtype.name) # convert the train_data to the same data type as the first element of the first element of self.nn.param and assign it to self.train_data + self.train_labels=train_labels.astype(self.nn.param[0][0].dtype.name) # convert the train_labels to the same data type as the first element of the first element of self.nn.param and assign it to self.train_labels + self.train_dataset=train_dataset # assign the train_dataset argument to the self.train_dataset attribute + self.test_data=test_data # assign the test_data argument to the self.test_data attribute + self.test_labels=test_labels # assign the test_labels argument to the self.test_labels attribute + self.test_dataset=test_dataset # assign the test_dataset argument to the self.test_dataset attribute + if test_data is not None or test_dataset is not None: # check if test_data or test_dataset is not None + self.test_flag=True # set the test_flag attribute to True + if train_data is not None: # check if train_data is not None + self.shape0=train_data.shape[0] # get the first dimension of train_data and assign it to self.shape0 + return # return from the method + + + def init(self): + # define a method for initializing some attributes + self.suspend=False # set the suspend attribute to False + self.stop=False # set the stop attribute to False + self.stop_flag=False # set the stop_flag attribute to False + self.save_epoch=None # set the save_epoch attribute to None + self.end_loss=None # set the end_loss attribute to None + self.end_acc=None # set the end_acc attribute to None + self.end_test_loss=None # set the end_test_loss attribute to None + self.end_test_acc=None # set the end_test_acc attribute to None + self.train_loss=None # set the train_loss attribute to None + self.train_acc=None # set the train_acc attribute to None + self.test_loss=None # set the test_loss attribute to None + self.test_acc=None # set the test_acc attribute to None + self.train_loss_list.clear() # clear the train_loss_list attribute + self.train_acc_list.clear() # clear the train_acc_list attribute + self.test_loss_list.clear() # clear the test_loss_list attribute + self.test_acc_list.clear() # clear the test_acc_list attribute + self.test_flag=False # set the test_flag attribute to False + self.train_counter=0 # set the train_counter attribute to 0 + self.epoch=0 # set the epoch attribute to 0 + self.total_epoch=0 # set the total_epoch attribute to 0 + self.time=0 # set the time attribute to 0 + self.total_time=0 # set the total_time attribute to 0 + return # return from the method + + + def end(self): + # define a method for checking if some conditions are met for ending training or testing process + if self.end_loss!=None and len(self.train_loss_list)!=0 and self.train_loss_list[-1]self.end_acc: # check if the end_acc attribute is not None and the train_acc_list attribute is not empty and the last element of the train_acc_list attribute is greater than the end_acc attribute + return True # return True + elif self.end_loss!=None and len(self.train_loss_list)!=0 and self.end_acc!=None and self.train_loss_list[-1]self.end_acc: # check if the end_loss attribute and the end_acc attribute are not None and the train_loss_list attribute and the train_acc_list attribute are not empty and the last element of the train_loss_list attribute is less than the end_loss attribute and the last element of the train_acc_list attribute is greater than the end_acc attribute + return True # return True + elif self.end_test_loss!=None and len(self.test_loss_list)!=0 and self.test_loss_list[-1]self.end_test_acc: # check if the end_test_acc attribute is not None and the test_acc_list attribute is not empty and the last element of the test_acc_list attribute is greater than the end_test_acc attribute + return True # return True + elif self.end_test_loss!=None and len(self.test_loss_list)!=0 and self.end_test_acc!=None and self.test_loss_list[-1]self.end_test_acc: # check if the end_test_loss attribute and the end_test_acc attribute are not None and the test_loss_list attribute and the test_acc_list attribute are not empty and the last element of the test_loss_list attribute is less than the end_test_loss attribute and the last element of the test_acc_list attribute is greater than the end_test_acc attribute + return True # return True + + + def loss_acc(self,output=None,labels_batch=None,loss=None,test_batch=None,total_loss=None,total_acc=None): + # define a method for calculating loss and accuracy for each batch + if self.batch!=None: # check if the batch attribute is not None + total_loss+=loss # add loss to total_loss + if hasattr(self.nn,'accuracy'): # check if the nn object has accuracy method + batch_acc=self.nn.accuracy(output,labels_batch) # call accuracy method with output and labels_batch as arguments and assign it to batch_acc + total_acc+=batch_acc # add batch_acc to total_acc + return total_loss,total_acc # return total_loss and total_acc + else: # check if the batch attribute is None + loss=loss.numpy() # convert loss to numpy array and assign it to loss + self.train_loss=loss # assign loss to self.train_loss + self.train_loss_list.append(loss) # append loss to self.train_loss_list + if hasattr(self.nn,'accuracy'): # check if the nn object has accuracy method + acc=self.nn.accuracy(output,self.train_labels) # call accuracy method with output and self.train_labels as arguments and assign it to acc + acc=acc.numpy() # convert acc to numpy array and assign it to acc + self.train_acc=acc # assign acc to self.train_acc + self.train_acc_list.append(acc) # append acc to self.train_acc_list + if self.test_flag==True: # check if the test_flag attribute is True + if hasattr(self.nn,'accuracy'): # check if the nn object has accuracy method + self.test_loss,self.test_acc=self.test(self.test_data,self.test_labels,test_batch) # call test method with self.test_data, self.test_labels, and test_batch as arguments and assign the results to self.test_loss and self.test_acc + self.test_loss_list.append(self.test_loss) # append self.test_loss to self.test_loss_list + self.test_acc_list.append(self.test_acc) # append self.test_acc to self.test_acc_list + else: # check if the nn object does not have accuracy method + self.test_loss=self.test(self.test_data,self.test_labels,test_batch) # call test method with self.test_data, self.test_labels, and test_batch as arguments and assign the result to self.test_loss + self.test_loss_list.append(self.test_loss) # append self.test_loss to self.test_loss_list + return # return from the method + + + def data_func(self,batch=None,index1=None,index2=None,j=None,flag=None): + # define a method for getting a batch of data and labels from train_data and train_labels attributes + if flag==None: # check if flag is None + if batch!=1: # check if batch is not 1 + data_batch=self.train_data[index1:index2] # slice the train_data attribute from index1 to index2 and assign it to data_batch + else: # check if batch is 1 + data_batch=self.train_data[j] # get the j-th element of train_data attribute and assign it to data_batch + if batch!=1: # check if batch is not 1 + labels_batch=self.train_labels[index1:index2] # slice the train_labels attribute from index1 to index2 and assign it to labels_batch + else: # check if batch is 1 + labels_batch=self.train_labels[j] # get the j-th element of train_labels attribute and assign it to labels_batch + else: # check if flag is not None + try: # try the following block of code + try: # try the following block of code + data_batch=self.platform.concat([self.train_data[index1:],self.train_data[:index2]],0) + # concatenate the train_data attribute from index1 to the end and from the beginning to index2 along the first axis and assign it to data_batch + labels_batch=self.platform.concat([self.train_labels[index1:],self.train_labels[:index2]],0) + # concatenate the train_labels attribute from index1 to the end and from the beginning to index2 along the first axis and assign it to labels_batch + except Exception: # handle any exception raised in the previous block of code + data_batch=np.concatenate([self.train_data[index1:],self.train_data[:index2]],0) + # concatenate the train_data attribute from index1 to the end and from the beginning to index2 along the first axis using numpy and assign it to data_batch + labels_batch=np.concatenate([self.train_labels[index1:],self.train_labels[:index2]],0) + # concatenate the train_labels attribute from index1 to the end and from the beginning to index2 along the first axis using numpy and assign it to labels_batch + except Exception as e: # handle any exception raised in the previous block of code and assign it to e + raise e # re-raise e + return data_batch,labels_batch # return data_batch and labels_batch + + + @function(jit_compile=True) + # use function decorator with jit_compile argument set to True for just-in-time compilation of this method for faster execution + def tf_opt(self,data,labels): + # define a method for optimizing parameters using tensorflow platform + try: # try the following block of code + try: # try the following block of code + with self.platform.GradientTape(persistent=True) as tape: # create a GradientTape object with persistent argument set to True and assign it to tape + try: # try the following block of code + output=self.nn.fp(data) # call fp method of the nn object with data as argument and assign it to output + loss=self.nn.loss(output,labels) # call loss method of the nn object with output and labels as arguments and assign it to loss + except Exception: # handle any exception raised in the previous block of code + output,loss=self.nn.fp(data,labels) # call fp method of the nn object with data and labels as arguments and assign the results to output and loss + except Exception: # handle any exception raised in the previous block of code + if hasattr(self.nn,'GradientTape'): # check if the nn object has GradientTape method + tape,output,loss=self.nn.GradientTape(data,labels) # call GradientTape method of the nn object with data and labels as arguments and assign the results to tape, output, and loss + except Exception as e: # handle any exception raised in the previous block of code and assign it to e + raise e # re-raise e + if hasattr(self.nn,'gradient'): # check if the nn object has gradient method + gradient=self.nn.gradient(tape,loss) # call gradient method of the nn object with tape and loss as arguments and assign it to gradient + else: # check if the nn object does not have gradient method + gradient=tape.gradient(loss,self.nn.param) # call gradient method of the tape object with loss and self.nn.param as arguments and assign it to gradient + if hasattr(self.nn.opt,'apply_gradients'): # check if the opt attribute of the nn object has apply_gradients method + self.nn.opt.apply_gradients(zip(gradient,self.nn.param)) # call apply_gradients method of the opt attribute of the nn object with zip object of gradient and self.nn.param as argument + else: # check if the opt attribute of the nn object does not have apply_gradients method + self.nn.opt(gradient) # call opt attribute of the nn object with gradient as argument + return output,loss # return output and loss + + + def pytorch_opt(self,data,labels): + # define a method for optimizing parameters using pytorch platform + output=self.nn.fp(data) # call fp method of the nn object with data as argument and assign it to output + loss=self.nn.loss(output,labels) # call loss method of the nn object with output and labels as arguments and assign it to loss + if hasattr(self.nn.opt,'zero_grad'): # check if the opt attribute of the nn object has zero_grad method + self.nn.opt.zero_grad() # call zero_grad method of the opt attribute of the nn object + loss.backward() # call backward method of the loss object + self.nn.opt.step() # call step method of the opt attribute of the nn object + else: # check if the opt attribute of the nn object does not have zero_grad method + self.nn.opt(loss) # call opt attribute of the nn object with loss as argument + return output,loss # return output and loss + + + def opt(self,data,labels): + # define a method for optimizing parameters using different platforms + if hasattr(self.platform,'DType'): + # check if the platform attribute has DType attribute, which indicates tensorflow platform + output,loss=self.tf_opt(data,labels) + # call tf_opt method with data and labels as arguments and assign the results to output and loss + else: + # check if the platform attribute does not have DType attribute, which indicates pytorch platform + output,loss=self.pytorch_opt(data,labels) + # call pytorch_opt method with data and labels as arguments and assign the results to output and loss + return output,loss + # return output and loss + + + def _train(self,batch=None,test_batch=None): + # define a private method for training on one epoch or iteration + if batch!=None: # check if batch is not None + total_loss=0 # initialize total_loss to 0 + total_acc=0 # initialize total_acc to 0 + if self.train_dataset!=None: # check if train_dataset attribute is not None + for data_batch,labels_batch in self.train_dataset: # iterate over the train_dataset attribute + if self.stop==True: # check if stop attribute is True + if self.stop_func(): # call stop_func method and check if it returns True + return # return from the method + if hasattr(self.nn,'data_func'): # check if the nn object has data_func method + data_batch,labels_batch=self.nn.data_func(data_batch,labels_batch) # call data_func method of the nn object with data_batch and labels_batch as arguments and assign the results to data_batch and labels_batch + output,batch_loss=self.opt(data_batch,labels_batch) # call opt method with data_batch and labels_batch as arguments and assign the results to output and batch_loss + total_loss,total_acc=self.loss_acc(output=output,labels_batch=labels_batch,loss=batch_loss,total_loss=total_loss,total_acc=total_acc) + # call loss_acc method with output, labels_batch, batch_loss, total_loss, and total_acc as arguments and assign the results to total_loss and total_acc + if hasattr(self.nn,'bc'): # check if the nn object has bc attribute + try: # try the following block of code + self.nn.bc.assign_add(1) # call assign_add method of the bc attribute of the nn object with 1 as argument + except Exception: # handle any exception raised in the previous block of code + self.nn.bc+=1 # add 1 to the bc attribute of the nn object + else: # check if train_dataset attribute is None + shape0=self.train_data.shape[0] # get the first dimension of train_data attribute and assign it to shape0 + batches=int((shape0-shape0%batch)/batch) # calculate the number of batches and assign it to batches + for j in range(batches): # iterate over batches + if self.stop==True: # check if stop attribute is True + if self.stop_func(): # call stop_func method and check if it returns True + return # return from the method + index1=j*batch # calculate the start index of a batch and assign it to index1 + index2=(j+1)*batch # calculate the end index of a batch and assign it to index2 + data_batch,labels_batch=self.data_func(batch,index1,index2,j) + # call data_func method with batch, index1, index2, and j as arguments and assign the results to data_batch and labels_batch + if hasattr(self.nn,'data_func'): + # check if the nn object has data_func method + data_batch,labels_batch=self.nn.data_func(data_batch,labels_batch) + # call data_func method of the nn object with data_batch and labels_batch as arguments and assign the results to data_batch and labels_batch + output,batch_loss=self.opt(data_batch,labels_batch) + # call opt method with data_batch and labels_batch as arguments and assign the results to output and batch_loss + total_loss,total_acc=self.loss_acc(output=output,labels_batch=labels_batch,loss=batch_loss,total_loss=total_loss,total_acc=total_acc) + # call loss_acc method with output, labels_batch, batch_loss, total_loss, and total_acc as arguments and assign the results to total_loss and total_acc + if hasattr(self.nn,'bc'): + # check if the nn object has bc attribute + try: + # try the following block of code + self.nn.bc.assign_add(1) + # call assign_add method of the bc attribute of the nn object with 1 as argument + except Exception: + # handle any exception raised in the previous block of code + self.nn.bc+=1 + # add 1 to the bc attribute of the nn object + if shape0%batch!=0: + # check if there is a remainder after dividing shape0 by batch + batches+=1 + # add 1 to batches + index1=batches*batch + # calculate the start index of a batch and assign it to index1 + index2=batch-(shape0-batches*batch) + # calculate the end index of a batch and assign it to index2 + try: + # try the following block of code + data_batch,labels_batch=self.data_func(batch,index1,index2,flag=True) + # call data_func method with batch, index1, index2, and flag set to True as arguments and assign the results to data_batch and labels_batch + if hasattr(self.nn,'data_func'): + # check if the nn object has data_func method + data_batch,labels_batch=self.nn.data_func(data_batch,labels_batch) + # call data_func method of the nn object with data_batch and labels_batch as arguments and assign the results to data_batch and labels_batch + output,batch_loss=self.opt(data_batch,labels_batch) + # call opt method with data_batch and labels_batch as arguments and assign the results to output and batch_loss + total_loss,total_acc=self.loss_acc(output=output,labels_batch=labels_batch,loss=batch_loss,total_loss=total_loss,total_acc=total_acc) + # call loss_acc method with output, labels_batch, batch_loss, total_loss, and total_acc as arguments and assign the results to total_loss and total_acc + if hasattr(self.nn,'bc'): + # check if the nn object has bc attribute + try: + # try the following block of code + self.nn.bc.assign_add(1) + # call assign_add method of the bc attribute of the nn object with 1 as argument + except Exception: + # handle any exception raised in the previous block of code + self.nn.bc+=1 + # add 1 to the bc attribute of the nn object + except Exception as e: + # handle any exception raised in the previous block of code and assign it to e + raise e # re-raise e + if hasattr(self.platform,'DType'): # check if the platform attribute has DType attribute, which indicates tensorflow platform + loss=total_loss.numpy()/batches # convert total_loss to numpy array, divide it by batches, and assign it to loss + else: # check if the platform attribute does not have DType attribute, which indicates pytorch platform + loss=total_loss.detach().numpy()/batches # detach total_loss from computation graph, convert it to numpy array, divide it by batches, and assign it to loss + if hasattr(self.nn,'accuracy'): # check if the nn object has accuracy method + train_acc=total_acc.numpy()/batches # convert total_acc to numpy array, divide it by batches, and assign it to train_acc + self.train_loss=loss # assign loss to self.train_loss + self.train_loss_list.append(loss) # append loss to self.train_loss_list + if hasattr(self.nn,'accuracy'): # check if the nn object has accuracy method + self.train_acc=train_acc # assign train_acc to self.train_acc + self.train_acc_list.append(train_acc) # append train_acc to self.train_acc_list + if self.test_flag==True: # check if the test_flag attribute is True + if hasattr(self.nn,'accuracy'): # check if the nn object has accuracy method + self.test_loss,self.test_acc=self.test(self.test_data,self.test_labels,test_batch) # call test method with self.test_data, self.test_labels, and test_batch as arguments and assign the results to self.test_loss and self.test_acc + self.test_loss_list.append(self.test_loss) # append self.test_loss to self.test_loss_list + self.test_acc_list.append(self.test_acc) # append self.test_acc to self.test_acc_list + else: # check if the nn object does not have accuracy method + self.test_loss=self.test(self.test_data,self.test_labels,test_batch) # call test method with self.test_data, self.test_labels, and test_batch as arguments and assign the result to self.test_loss + self.test_loss_list.append(self.test_loss) # append self.test_loss to self.test_loss_list + else: # check if batch is None + output,train_loss=self.opt(self.train_data,self.train_labels) # call opt method with self.train_data and self.train_labels as arguments and assign the results to output and train_loss + self.loss_acc(output=output,labels_batch=labels_batch,loss=train_loss,test_batch=test_batch,total_loss=total_loss,total_acc=total_acc) + # call loss_acc method with output, labels_batch, train_loss, test_batch, total_loss, and total_acc as arguments + return # return from the method + + + def train(self,batch=None,epoch=None,test_batch=None,save=None,one=True,p=None,s=None): + # define a method for training on multiple epochs or iterations + self.batch=batch # assign batch argument to batch attribute + self.epoch=0 # set epoch attribute to 0 + self.train_counter+=1 # add 1 to train_counter attribute + if p==None: # check if p argument is None + self.p=9 # set p attribute to 9 + else: # check if s argument is not None + self.p=p-1 # subtract 1 from p argument and assign it to p attribute + if s==None: # check if s argument is None + self.s=1 # set s attribute to 1 + self.file_list=None # set file_list attribute to None + else: # check if s argument is not None + self.s=s-1 # subtract 1 from s argument and assign it to s attribute + self.file_list=[] # set file_list attribute to an empty list + if epoch!=None: # check if epoch argument is not None + for i in range(epoch): # iterate over epoch argument + t1=time.time() # get the current time and assign it to t1 + self._train(batch,test_batch) # call _train method with batch and test_batch as arguments + if self.stop_flag==True: # check if stop_flag attribute is True + return # return from the method + if hasattr(self.nn,'ec'): # check if the nn object has ec attribute + try: # try the following block of code + self.nn.ec.assign_add(1) # call assign_add method of the ec attribute of the nn object with 1 as argument + except Exception: # handle any exception raised in the previous block of code + self.nn.ec+=1 # add 1 to the ec attribute of the nn object + self.total_epoch+=1 # add 1 to total_epoch attribute + if epoch%10!=0: # check if epoch argument is not divisible by 10 + p=epoch-epoch%self.p # calculate the nearest multiple of p attribute that is less than or equal to epoch argument and assign it to p + p=int(p/self.p) # divide p by p attribute and convert it to integer and assign it to p + s=epoch-epoch%self.s # calculate the nearest multiple of s attribute that is less than or equal to epoch argument and assign it to s + s=int(s/self.s) # divide s by s attribute and convert it to integer and assign it to s + else: # check if epoch argument is divisible by 10 + p=epoch/(self.p+1) # divide epoch argument by p attribute plus 1 and assign it to p + p=int(p) # convert p to integer and assign it to p + s=epoch/(self.s+1) # divide epoch argument by s attribute plus 1 and assign it to s + s=int(s) # convert s to integer and assign it to s + if p==0: # check if p is 0 + p=1 # set p to 1 + if s==0: # check if s is 0 + s=1 # set s to 1 + if i%p==0: # check if i is divisible by p + if self.test_flag==False: + # check if test_flag attribute is False + if hasattr(self.nn,'accuracy'): + # check if the nn object has accuracy method + print('epoch:{0} loss:{1:.6f}'.format(i+1,self.train_loss)) + # print the epoch number and train_loss attribute with six decimal places + if self.acc_flag=='%': + # check if acc_flag attribute is '%' + print('epoch:{0} accuracy:{1:.1f}'.format(i+1,self.train_acc*100)) + # print the epoch number and train_acc attribute multiplied by 100 with one decimal place + else: + # check if acc_flag attribute is not '%' + print('epoch:{0} accuracy:{1:.6f}'.format(i+1,self.train_acc)) + # print the epoch number and train_acc attribute with six decimal places + print() + # print a blank line + else: + # check if the nn object does not have accuracy method + print('epoch:{0} loss:{1:.6f}'.format(i+1,self.train_loss)) + # print the epoch number and train_loss attribute with six decimal places + print() + # print a blank line + else: + # check if test_flag attribute is True + if hasattr(self.nn,'accuracy'): + # check if the nn object has accuracy method + print('epoch:{0} loss:{1:.6f},test loss:{2:.6f}'.format(i+1,self.train_loss,self.test_loss)) + # print the epoch number, train_loss attribute, and test_loss attribute with six decimal places + if self.acc_flag=='%': + # check if acc_flag attribute is '%' + print('epoch:{0} accuracy:{1:.1f},test accuracy:{2:.1f}'.format(i+1,self.train_acc*100,self.test_acc*100)) + # print the epoch number, train_acc attribute multiplied by 100, and test_acc attribute multiplied by 100 with one decimal place + else: + # check if acc_flag attribute is not '%' + print('epoch:{0} accuracy:{1:.1f},test accuracy:{2:.1f}'.format(i+1,self.train_acc,self.test_acc)) + # print the epoch number, train_acc attribute, and test_acc attribute with one decimal place + print() + # print a blank line + else: + # check if the nn object does not have accuracy method + print('epoch:{0} loss:{1:.6f},test loss:{2:.6f}'.format(i+1,self.train_loss,self.test_loss)) + # print the epoch number, train_loss attribute, and test_loss attribute with six decimal places + print() + # print a blank line + if save!=None and i%s==0: # check if save argument is not None and i is divisible by s + self.save(self.total_epoch,one) # call save method with total_epoch attribute and one argument as arguments + t2=time.time() # get the current time and assign it to t2 + self.time+=(t2-t1) # subtract t1 from t2 and add it to time attribute + else: # check if epoch argument is None + i=0 # set i to 0 + while True: # start an infinite loop + t1=time.time() # get the current time and assign it to t1 + self._train(test_batch=test_batch) # call _train method with test_batch as argument + if self.stop_flag==True: # check if stop_flag attribute is True + return # return from the method + i+=1 # add 1 to i + if hasattr(self.nn,'ec'): # check if the nn object has ec attribute + try: # try the following block of code + self.nn.ec.assign_add(1) # call assign_add method of the ec attribute of the nn object with 1 as argument + except Exception: # handle any exception raised in the previous block of code + self.nn.ec+=1 # add 1 to the ec attribute of the nn object + self.total_epoch+=1 # add 1 to total_epoch attribute + if epoch%10!=0: # check if epoch argument is not divisible by 10 + p=epoch-epoch%self.p # calculate the nearest multiple of p attribute that is less than or equal to epoch argument and assign it to p + p=int(p/self.p) # divide p by p attribute and convert it to integer and assign it to p + s=epoch-epoch%self.s # calculate the nearest multiple of s attribute that is less than or equal to epoch argument and assign it to s + s=int(s/self.s) # divide s by s attribute and convert it to integer and assign it to s + else: # check if epoch argument is divisible by 10 + p=epoch/(self.p+1) # divide epoch argument by p attribute plus 1 and assign it to p + p=int(p) # convert p to integer and assign it to p + s=epoch/(self.s+1) # divide epoch argument by s attribute plus 1 and assign it to s + s=int(s) # convert s to integer and assign it to s + if p==0: # check if p is 0 + p=1 # set p to 1 + if s==0: # check if s is 0 + s=1 # set s to 1 + if i%p==0: # check if i is divisible by p + if self.test_flag==False: + # check if test_flag attribute is False + if hasattr(self.nn,'accuracy'): + # check if the nn object has accuracy method + print('epoch:{0} loss:{1:.6f}'.format(i+1,self.train_loss)) + # print the epoch number and train_loss attribute with six decimal places + if self.acc_flag=='%': + # check if acc_flag attribute is '%' + print('epoch:{0} accuracy:{1:.1f}'.format(i+1,self.train_acc*100)) + # print the epoch number and train_acc attribute multiplied by 100 with one decimal place + else: + # check if acc_flag attribute is not '%' + print('epoch:{0} accuracy:{1:.6f}'.format(i+1,self.train_acc)) + # print the epoch number and train_acc attribute with six decimal places + print() + # print a blank line + else: + # check if the nn object does not have accuracy method + print('epoch:{0} loss:{1:.6f}'.format(i+1,self.train_loss)) + # print the epoch number and train_loss attribute with six decimal places + print() + # print a blank line + else: + # check if test_flag attribute is True + if hasattr(self.nn,'accuracy'): + # check if the nn object has accuracy method + print('epoch:{0} loss:{1:.6f},test loss:{2:.6f}'.format(i+1,self.train_loss,self.test_loss)) + # print the epoch number, train_loss attribute, and test_loss attribute with six decimal places + if self.acc_flag=='%': + # check if acc_flag attribute is '%' + print('epoch:{0} accuracy:{1:.1f},test accuracy:{2:.1f}'.format(i+1,self.train_acc*100,self.test_acc*100)) + # print the epoch number, train_acc attribute multiplied by 100, and test_acc attribute multiplied by 100 with one decimal place + else: + # check if acc_flag attribute is not '%' + print('epoch:{0} accuracy:{1:.1f},test accuracy:{2:.1f}'.format(i+1,self.train_acc,self.test_acc)) + # print the epoch number, train_acc attribute, and test_acc attribute with one decimal place + print() + # print a blank line + else: + # check if the nn object does not have accuracy method + print('epoch:{0} loss:{1:.6f},test loss:{2:.6f}'.format(i+1,self.train_loss,self.test_loss)) + # print the epoch number, train_loss attribute, and test_loss attribute with six decimal places + print() + # print a blank line + if save!=None and i%s==0: # check if save argument is not None and i is divisible by s + self.save(self.total_epoch,one) # call save method with total_epoch attribute and one argument as arguments + t2=time.time() # get the current time and assign it to t2 + self.time+=(t2-t1) # subtract t1 from t2 and add it to time attribute + if save!=None: # check if save argument is not None + self.save() # call save method + self._time=self.time-int(self.time) # subtract the integer part of time attribute from time attribute and assign it to _time attribute + if self._time<0.5: # check if _time attribute is less than 0.5 + self.time=int(self.time) # convert time attribute to integer and assign it to time attribute + else: # check if _time attribute is not less than 0.5 + self.time=int(self.time)+1 # convert time attribute to integer, add 1, and assign it to time attribute + self.total_time+=self.time # add time attribute to total_time attribute + if self.test_flag==False: # check if test_flag attribute is False + print('last loss:{0:.6f}'.format(self.train_loss)) # print the last train_loss attribute with six decimal places + else: # check if test_flag attribute is True + print('last loss:{0:.6f},last test loss:{1:.6f}'.format(self.train_loss,self.test_loss)) # print the last train_loss attribute and last test_loss attribute with six decimal places + if hasattr(self.nn,'accuracy'): # check if the nn object has accuracy method + if self.acc_flag=='%': # check if acc_flag attribute is '%' + if self.test_flag==False: # check if test_flag attribute is False + print('last accuracy:{0:.1f}'.format(self.train_acc*100)) # print the last train_acc attribute multiplied by 100 with one decimal place + else: # check if test_flag attribute is True + print('last accuracy:{0:.1f},last test accuracy:{1:.1f}'.format(self.train_acc*100,self.test_acc*100)) # print the last train_acc attribute multiplied by 100 and last test_acc attribute multiplied by 100 with one decimal place + else: # check if acc_flag attribute is not '%' + if self.test_flag==False: # check if test_flag attribute is False + print('last accuracy:{0:.6f}'.format(self.train_acc)) # print the last train_acc attribute with six decimal places + else: # check if test_flag attribute is True + print('last accuracy:{0:.6f},last test accuracy:{1:.6f}'.format(self.train_acc,self.test_acc)) # print the last train_acc attribute and last test_acc attribute with six decimal places + print() # print a blank line + print('time:{0}s'.format(self.time)) # print the time attribute with 's' as unit + self.training_flag=False # set training_flag attribute to False + return # return from the method + + + def train_online(self): + # define a method for online training + while True: # start an infinite loop + if hasattr(self.nn,'save'): # check if the nn object has save method + self.nn.save(self.save) # call save method of the nn object with save attribute as argument + if hasattr(self.nn,'stop_flag'): # check if the nn object has stop_flag attribute + if self.nn.stop_flag==True: # check if stop_flag attribute of the nn object is True + return # return from the method + if hasattr(self.nn,'stop_func'): # check if the nn object has stop_func method + if self.nn.stop_func(): # call stop_func method of the nn object and check if it returns True + return # return from the method + if hasattr(self.nn,'suspend_func'): # check if the nn object has suspend_func method + self.nn.suspend_func() # call suspend_func method of the nn object + data=self.nn.online() # call online method of the nn object and assign it to data + if data=='stop': # check if data is 'stop' + return # return from the method + elif data=='suspend': # check if data is 'suspend' + self.nn.suspend_func() # call suspend_func method of the nn object + output,loss=self.opt(data[0],data[1]) # call opt method with data[0] and data[1] as arguments and assign the results to output and loss + loss=loss.numpy() # convert loss to numpy array and assign it to loss + if len(self.nn.train_loss_list)==self.nn.max_length: # check if the length of train_loss_list attribute of the nn object is equal to max_length attribute of the nn object + del self.nn.train_loss_list[0] # delete the first element of train_loss_list attribute of the nn object + self.nn.train_loss_list.append(loss) # append loss to train_loss_list attribute of the nn object + try: # try the following block of code + if hasattr(self.nn,'accuracy'): # check if the nn object has accuracy method + train_acc=self.nn.accuracy(output,data[1]) # call accuracy method of the nn object with output and data[1] as arguments and assign it to train_acc + if len(self.nn.train_acc_list)==self.nn.max_length: # check if the length of train_acc_list attribute of the nn object is equal to max_length attribute of the nn object + del self.nn.train_acc_list[0] # delete the first element of train_acc_list attribute of the nn object + self.train_acc_list.append(train_acc) # append train_acc to train_acc_list attribute of the nn object + except Exception as e: # handle any exception raised in the previous block of code and assign it to e + raise e # re-raise e + if hasattr(self.nn,'counter'): # check if the nn object has counter attribute + self.nn.counter+=1 # add 1 to counter attribute of the nn object + return # return from the method + + + @function(jit_compile=True) + # use function decorator with jit_compile argument set to True for just-in-time compilation of this method for faster execution + def test_tf(self,data,labels): + # define a method for testing using tensorflow platform + try: + # try the following block of code + try: + # try the following block of code + output=self.nn.fp(data) + # call fp method of the nn object with data as argument and assign it to output + loss=self.nn.loss(output,labels) + # call loss method of the nn object with output and labels as arguments and assign it to loss + except Exception: + # handle any exception raised in the previous block of code + output,loss=self.nn.fp(data,labels) + # call fp method of the nn object with data and labels as arguments and assign the results to output and loss + except Exception as e: + # handle any exception raised in the previous block of code and assign it to e + raise e # re-raise e + try: # try the following block of code + acc=self.nn.accuracy(output,labels) # call accuracy method of the nn object with output and labels as arguments and assign it to acc + except Exception as e: # handle any exception raised in the previous block of code and assign it to e + if hasattr(self.nn,'accuracy'): # check if the nn object has accuracy method + raise e # re-raise e + else: # check if the nn object does not have accuracy method + acc=None # set acc to None + return loss,acc # return loss and acc + + + def test_pytorch(self,data,labels): + # define a method for testing using pytorch platform + output=self.nn.fp(data) # call fp method of the nn object with data as argument and assign it to output + loss=self.nn.loss(output,labels) # call loss method of the nn object with output and labels as arguments and assign it to loss + try: # try the following block of code + acc=self.nn.accuracy(output,labels) # call accuracy method of the nn object with output and labels as arguments and assign it to acc + except Exception as e: # handle any exception raised in the previous block of code and assign it to e + if hasattr(self.nn,'accuracy'): # check if the nn object has accuracy method + raise e # re-raise e + else: # check if the nn object does not have accuracy method + acc=None # set acc to None + return loss,acc # return loss and acc + + + def test(self,test_data=None,test_labels=None,batch=None): + # define a method for testing on a given data set + if test_data is not None and type(self.nn.param[0])!=list: + # check if test_data is not None and the type of the first element of self.nn.param is not list + test_data=test_data.astype(self.nn.param[0].dtype.name) + # convert the test_data to the same data type as the first element of self.nn.param and assign it to test_data + test_labels=test_labels.astype(self.nn.param[0].dtype.name) + # convert the test_labels to the same data type as the first element of self.nn.param and assign it to test_labels + elif test_data is not None: + # check if test_data is not None + test_data=test_data.astype(self.nn.param[0][0].dtype.name) + # convert the test_data to the same data type as the first element of the first element of self.nn.param and assign it to test_data + test_labels=test_labels.astype(self.nn.param[0][0].dtype.name) + # convert the test_labels to the same data type as the first element of the first element of self.nn.param and assign it to test_labels + if self.process_t!=None: + # check if process_t attribute is not None + if self.prefetch_batch_size_t==None: + # check if prefetch_batch_size_t attribute is None + parallel_test_=parallel_test(self.nn,self.test_data,self.test_labels,self.process_t,batch,test_dataset=self.test_dataset) + # create a parallel_test object with self.nn, self.test_data, self.test_labels, self.process_t, batch, and self.test_dataset as arguments and assign it to parallel_test_ + else: + # check if prefetch_batch_size_t attribute is not None + parallel_test_=parallel_test(self.nn,self.test_data,self.test_labels,self.process_t,batch,self.prefetch_batch_size_t,self.test_dataset) + # create a parallel_test object with self.nn, self.test_data, self.test_labels, self.process_t, batch, self.prefetch_batch_size_t, and self.test_dataset as arguments and assign it to parallel_test_ + if type(self.test_data)!=list: + # check if the type of test_data attribute is not list + parallel_test_.segment_data() + # call segment_data method of parallel_test_ object + for p in range(self.process_t): + # iterate over process_t attribute + Process(target=parallel_test_.test).start() + # create a Process object with target argument set to test method of parallel_test_ object and call start method on it + try: + # try the following block of code + if hasattr(self.nn,'accuracy'): + # check if the nn object has accuracy method + test_loss,test_acc=parallel_test_.loss_acc() + # call loss_acc method of parallel_test_ object and assign the results to test_loss and test_acc except Exception as e: - try: - if self.nn.accuracy!=None: # if the neural network has an accuracy method defined - raise e # raise the exception - except Exception: - pass - if self.test_flag==True: # if test data is used - self.test_loss,self.test_acc=self.test(self.test_data,self.test_labels,test_batch) # calculate the test loss and accuracy values using the test method - self.test_loss_list.append(self.test_loss) # append the test loss value to test loss list - try: - self.test_acc_list.append(self.test_acc) # append the test accuracy value to test accuracy list - except Exception as e: - try: - if self.nn.accuracy!=None: # if the neural network has an accuracy method defined - raise e # raise the exception - except Exception: - pass - else: # if batch mode is not used (use all data at once) - output,train_loss=self.opt(self.train_data,self.train_labels) # perform one optimization step using the opt method and get the output and train loss values - self.loss_acc(output=output,labels_batch=labels_batch,loss=train_loss,test_batch=test_batch,total_loss=total_loss,total_acc=total_acc) # calculate and update the train and test loss and accuracy values using the loss_acc method - return # return nothing for this case - - - def train(self,batch=None,epoch=None,test_batch=None,save=None,one=True,p=None,s=None): # a method to perform multiple epochs of training - self.batch=batch # assign the batch size for training data to batch attribute - self.epoch=0 # initialize the current epoch number to 0 - self.train_counter+=1 # increase the train counter by 1 - if p==None: # if p is None, it means the default value of p is used - self.p=9 # assign 9 to p attribute, which means print the train and test loss and accuracy values every 10 epochs - else: - self.p=p-1 # assign p-1 to p attribute, which means print the train and test loss and accuracy values every p epochs - if s==None: # if s is None, it means the default value of s is used - self.s=1 # assign 1 to s attribute, which means save the model every epoch - self.file_list=None # assign None to file_list attribute, which means do not keep track of saved files - else: - self.s=s-1 # assign s-1 to s attribute, which means save the model every s epochs - self.file_list=[] # assign an empty list to file_list attribute, which means keep track of saved files - if epoch!=None: # if epoch is not None, it means a fixed number of epochs is given - for i in range(epoch): # iterate over each epoch index - t1=time.time() # record the start time of this epoch - self._train(batch,test_batch) # perform one epoch of training using the _train method - if self.stop_flag==True: # if the stop flag is set to True, it means the training has reached the target loss or accuracy values and has been stopped - return # return nothing and end the training - try: - try: - self.nn.ec.assign_add(1) # use the neural network's epoch counter's assign_add method to increase the epoch counter by 1 (for tensorflow platform) - except Exception: # if the neural network does not have an epoch counter or its assign_add method fails - self.nn.ec+=1 # use the normal addition operation to increase the epoch counter by 1 (for pytorch platform) - except Exception: - pass - self.total_epoch+=1 # increase the total epoch number by 1 for all trainings - if epoch%10!=0: - p=epoch-epoch%self.p - p=int(p/self.p) - s=epoch-epoch%self.s - s=int(s/self.s) - else: - p=epoch/(self.p+1) - p=int(p) - s=epoch/(self.s+1) - s=int(s) - if p==0: - p=1 - if s==0: - s=1 - if i%p==0: # if this epoch index is a multiple of p (or p+1) - if self.test_flag==False: # if test data is not used - try: - if self.nn.accuracy!=None: # if the neural network has an accuracy method defined - print('epoch:{0} loss:{1:.6f}'.format(i+1,self.train_loss)) # print the current epoch number and train loss value - if self.acc_flag=='%': # if percentage mode is used for accuracy display - print('epoch:{0} accuracy:{1:.1f}'.format(i+1,self.train_acc*100)) # print the current epoch number and train accuracy value in percentage - else: # if decimal mode is used for accuracy display - print('epoch:{0} accuracy:{1:.6f}'.format(i+1,self.train_acc)) # print the current epoch number and train accuracy value in print('epoch:{0} accuracy:{1:.6f}'.format(i+1,self.train_acc)) # print the current epoch number and train accuracy value in decimal - print() # print an empty line for readability - except Exception: # if the neural network does not have an accuracy method defined or it fails - print('epoch:{0} loss:{1:.6f}'.format(i+1,self.train_loss)) # print the current epoch number and train loss value - print() # print an empty line for readability - else: # if test data is used - try: - if self.nn.accuracy!=None: # if the neural network has an accuracy method defined - print('epoch:{0} loss:{1:.6f},test loss:{2:.6f}'.format(i+1,self.train_loss,self.test_loss)) # print the current epoch number, train loss value and test loss value - if self.acc_flag=='%': # if percentage mode is used for accuracy display - print('epoch:{0} accuracy:{1:.1f},test accuracy:{2:.1f}'.format(i+1,self.train_acc*100,self.test_acc*100)) # print the current epoch number, train accuracy value and test accuracy value in percentage - else: # if decimal mode is used for accuracy display - print('epoch:{0} accuracy:{1:.6f},test accuracy:{2:.6f}'.format(i+1,self.train_acc,self.test_acc)) # print the current epoch number, train accuracy value and test accuracy value in decimal - print() # print an empty line for readability - except Exception: # if the neural network does not have an accuracy method defined or it fails - print('epoch:{0} loss:{1:.6f},test loss:{2:.6f}'.format(i+1,self.train_loss,self.test_loss)) # print the current epoch number, train loss value and test loss value - print() # print an empty line for readability - if save!=None and i%s==0: # if save is not None, it means a file name is given to save the model, and if this epoch index is a multiple of s (or s+1) - self.save(self.total_epoch,one) # save the model using the save method with total epoch number and one flag as inputs - t2=time.time() # record the end time of this epoch - self.time+=(t2-t1) # calculate and update the time elapsed for this training session - else: # if epoch is None, it means an infinite number of epochs is given - i=0 # initialize the epoch index to 0 - while True: # loop indefinitely until stopped by other conditions - t1=time.time() # record the start time of this epoch - self._train(test_batch=test_batch) # perform one epoch of training using the _train method - if self.stop_flag==True: # if the stop flag is set to True, it means the training has reached the target loss or accuracy values and has been stopped - return # return nothing and end the training - i+=1 # increase the epoch index by 1 - try: - try: - self.nn.ec.assign_add(1) # use the neural network's epoch counter's assign_add method to increase the epoch counter by 1 (for tensorflow platform) - except Exception: # if the neural network does not have an epoch counter or its assign_add method fails - self.nn.ec+=1 # use the normal addition operation to increase the epoch counter by 1 (for pytorch platform) - except Exception: - pass - self.total_epoch+=1 # increase the total epoch number by 1 for all trainings - if epoch%10!=0: - p=epoch-epoch%self.p - p=int(p/self.p) - s=epoch-epoch%self.s - s=int(s/self.s) - else: - p=epoch/(self.p+1) - p=int(p) - s=epoch/(self.s+1) - s=int(s) - if p==0: - p=1 - if s==0: - s=1 - if i%p==0: # if this epoch index is a multiple of p (or p+1) - if self.test_flag==False: # if test data is not used - try: - if self.nn.accuracy!=None: # if the neural network has an accuracy method defined - print('epoch:{0} loss:{1:.6f}'.format(i+1,self.train_loss)) # print the current epoch number and train loss value - if self.acc_flag=='%': # if percentage mode is used for accuracy display - print('epoch:{0} accuracy:{1:.1f}'.format(i+1,self.train_acc*100)) # print the current epoch number and train accuracy value in percentage - else: # if decimal mode is used for accuracy display - print('epoch:{0} accuracy:{1:.6f}'.format(i+1,self.train_acc)) # print the current epoch number and train accuracy value in decimal - print() # print an empty line for readability - except Exception: # if the neural network does not have an accuracy method defined or it fails - print('epoch:{0} loss:{1:.6f}'.format(i+1,self.train_loss)) # print the current epoch number and train loss value - print() # print an empty line for readability - else: # if test data is used - try: - if self.nn.accuracy!=None: # if the neural network has an accuracy method defined - print('epoch:{0} loss:{1:.6f},test loss:{2:.6f}'.format(i+1,self.train_loss,self.test_loss)) # print the current epoch number, train loss value and test loss value - if self.acc_flag=='%': # if percentage mode is used for accuracy display - print('epoch:{0} accuracy:{1:.1f},test accuracy:{2:.1f}'.format(i+1,self.train_acc*100,self.test_acc*100)) # print the current epoch number, train accuracy value and test accuracy value in percentage - else: # if decimal mode is used for accuracy display - print('epoch:{0} accuracy:{1:.6f},test accuracy:{2:.6f}'.format(i+1,self.train_acc,self.test_acc)) # print the current epoch number, train accuracy value and test accuracy value in decimal - print() # print an empty line for readability - except Exception: # if the neural network does not have an accuracy method defined or it fails - print('epoch:{0} loss:{1:.6f},test loss:{2:.6f}'.format(i+1,self.train_loss,self.test_loss)) # print the current epoch number, train loss value and test loss value - print() # print an empty line for readability - if save!=None and i%s==0: # if save is not None, it means a file name is given to save the model, and if this epoch index is a multiple of s (or s+1) - self.save(self.total_epoch,one) # save the model using the save method with total epoch number and one flag as inputs - t2=time.time() # record the end time of this epoch - self.time+=(t2-t1) # calculate and update the time elapsed for this training session - if save!=None: # if save is not None, it means a file name is given to save the model - self.save() # save the model using the save method without any inputs (use default values) - self._time=self.time-int(self.time) # get the fractional part of the time elapsed for this training session - if self._time<0.5: # if the fractional part is less than 0.5 - self.time=int(self.time) # round down the time elapsed to integer - else: # if the fractional part is greater than or equal to 0.5 - self.time=int(self.time)+1 # round up the time elapsed to integer - self.total_time+=self.time # calculate and update the total time elapsed for all trainings - if self.test_flag==False: # if test data is not used - print('last loss:{0:.6f}'.format(self.train_loss)) # print the last train loss value - else: # if test data is used - print('last loss:{0:.6f},last test loss:{1:.6f}'.format(self.train_loss,self.test_loss)) # print the last train loss value and test loss value - try: - if self.nn.accuracy!=None: # if the neural network has an accuracy method defined - if self.acc_flag=='%': # if percentage mode is used for accuracy display - if self.test_flag==False: # if test data is not used - print('last accuracy:{0:.1f}'.format(self.train_acc*100)) # print the last train accuracy value in percentage - else: # if test else: # if test data is used - print('last accuracy:{0:.1f},last test accuracy:{1:.1f}'.format(self.train_acc*100,self.test_acc*100)) # print the last train accuracy value and test accuracy value in percentage - else: # if decimal mode is used for accuracy display - if self.test_flag==False: # if test data is not used - print('last accuracy:{0:.6f}'.format(self.train_acc)) # print the last train accuracy value in decimal - else: # if test data is used - print('last accuracy:{0:.6f},last test accuracy:{1:.6f}'.format(self.train_acc,self.test_acc)) # print the last train accuracy value and test accuracy value in decimal - except Exception: # if the neural network does not have an accuracy method defined or it fails - pass - print() # print an empty line for readability - print('time:{0}s'.format(self.time)) # print the time elapsed for this training session - self.training_flag=False # set the training flag to False, which means the training is finished - return # return nothing and end the training - - - def train_ol(self): # a method to perform online learning, which means updating the model with one sample at a time - while True: # loop indefinitely until stopped by other conditions - if self.stop_flag==True: # if the stop flag is set to True, it means the online learning has reached the target loss or accuracy values and has been stopped - return # return nothing and end the online learning - if self.save_flag==True: # if the save flag is set to True, it means a request to save the model has been made - self.save() # save the model using the save method without any inputs (use default values) - self.suspend_func() # check whether a request to suspend the online learning has been made using the suspend_func method - data=self.nn.ol() # get one sample of data and label from the neural network's ol method, which should be defined by the user - if data=='stop': # if the data is 'stop', it means a request to stop the online learning has been made - return # return nothing and end the online learning - elif data=='suspend': # if the data is 'suspend', it means a request to suspend the online learning has been made - self.nn.suspend=True # set the neural network's suspend attribute to True, which means the online learning is suspended - while True: # loop indefinitely until resumed by other conditions - if self.nn.suspend==False: # if the neural network's suspend attribute is set to False, it means a request to resume the online learning has been made - break # break the loop and resume the online learning - continue # continue to the next iteration of online learning - output,loss=self.opt(data[0],data[1]) # perform one optimization step using the opt method and get the output and loss values - loss=loss.numpy() # convert the loss value to numpy array - if len(self.nn.train_loss_list)==self.nn.max_length: # if the train loss list has reached its maximum length, which should be defined by the user - del self.nn.train_loss_list[0] # delete the first element of train loss list - self.nn.train_loss_list.append(loss) # append the loss value to train loss list - try: - train_acc=self.nn.accuracy(output,data[1]) # calculate the accuracy value using the neural network's accuracy method - if len(self.nn.train_acc_list)==self.nn.max_length: # if the train accuracy list has reached its maximum length, which should be defined by the user - del self.nn.train_acc_list[0] # delete the first element of train accuracy list - self.train_acc_list.append(train_acc) # append the accuracy value to train accuracy list - except Exception as e: - try: - if self.nn.accuracy!=None: # if the neural network has an accuracy method defined - raise e # raise the exception - except Exception: - pass - try: - self.nn.c+=1 # increase the neural network's counter by 1, which should be defined by the user to keep track of how many samples have been used for online learning - except Exception: - pass - return # return nothing and end the online learning - - - def test(self,test_data=None,test_labels=None,batch=None): # a method to calculate the test loss and accuracy values using test data and labels - if batch!=None: # if batch mode is used - total_loss=0 # initialize the total loss value for all batches - total_acc=0 # initialize the total accuracy value for all batches - if self.test_dataset!=None: # if a tensorflow or pytorch dataset object is used for test data - batches=0 - for data_batch,labels_batch in self.test_dataset: # iterate over each batch of data and labels from the test dataset - batches+=1 - output=self.nn.fp(data_batch) # get the output value using the neural network's forward propagation method - batch_loss=self.nn.loss(output,labels_batch) # get the batch loss value using the neural network's loss function - total_loss+=batch_loss # accumulate the batch loss to total loss - try: - batch_acc=self.nn.accuracy(output,labels_batch) # calculate the batch accuracy using the neural network's accuracy method - total_acc+=batch_acc # accumulate the batch accuracy to total accuracy - except Exception as e: - try: - if self.nn.accuracy!=None: # if the neural network has an accuracy method defined - raise e # raise the exception - except Exception: - pass - test_loss=total_loss.numpy()/batches # convert the total loss value to numpy array and divide it by number of batches to get the average loss value for test data - try: - if self.nn.accuracy!=None: # if the neural network has an accuracy method defined - test_acc=total_acc.numpy()/batches # convert the total accuracy value to numpy array and divide it by number of batches to get the average accuracy value for test data - except Exception: - pass - else: # if a numpy array is used for test data - batches=int((test_data.shape[0]-test_data.shape[0]%batch)/batch) # calculate how many batches are needed for test data according to batch size - shape0=test_data.shape[0] # get the number of samples in test data - for j in range(batches): # iterate over each batch index - index1=j*batch # calculate the start index of test data for this batch - index2=(j+1)*batch # calculate the end index of test data for this batch - data_batch=test_data[index1:index2] # get a slice of test data according to the index range - labels_batch=test_labels[index1:index2] # get a slice of test labels according to the index range - output=self.nn.fp(data_batch) # get the output value using the neural network's forward propagation method - batch_loss=self.nn.loss(output,labels_batch) # get the batch loss value using the neural network's loss function - total_loss+=batch_loss # accumulate the batch loss to total loss - try: - batch_acc=self.nn.accuracy(output,labels_batch) # calculate the batch accuracy using the neural network's accuracy method - total_acc+=batch_acc # accumulate the batch accuracy to total accuracy - except Exception as e: - try: - if self.nn.accuracy!=None: # if the neural network has an accuracy method defined - raise e # raise the exception - except Exception: - pass - if shape0%batch!=0: # if there are some samples left in test data that are not enough to form a full batch - batches+=1 # increase the number of batches by 1 to include the remaining samples - index1=batches*batch # calculate the start index of test data for the last batch - index2=batch-(shape0-batches*batch) # calculate how many samples are needed from the beginning of test data to form a full batch - try: - try: - data_batch=self.platform.concat([test_data[index1:],test_data[:index2]],0) # concatenate two slices of test data from the end and the beginning to form a batch - labels_batch=self.platform.concat([test_labels[index1:],test_labels[:index2]],0) # concatenate two slices of test labels from the end and the beginning to form a batch - except Exception: # if the platform's concat method fails - data_batch=np.concatenate([test_data[index1:],test_data[:index2]],0) # use numpy's concatenate method instead - labels_batch=np.concatenate([test_labels[index1:],test_labels[:index2]],0) # use numpy's concatenate method instead - except Exception as e: - raise e # raise any other exception - output=self.nn.fp(data_batch) # get the output value using the neural network's forward propagation method - batch_loss=self.nn.loss(output,labels_batch) # get the batch loss value using the neural network's loss function - total_loss+=batch_loss # accumulate the batch loss to total loss - try: - batch_acc=self.nn.accuracy(output,labels_batch) # calculate the batch accuracy using the neural network's accuracy method - total_acc+=batch_acc # accumulate the batch accuracy to total accuracy - except Exception as e: - try: - if self.nn.accuracy!=None: # if the neural network has an accuracy method defined - raise e # raise the exception - except Exception: - pass - test_loss=total_loss.numpy()/batches # convert the total loss value to numpy array and divide it by number of batches to get the average loss value for test data - try: - if self.nn.accuracy!=None: # if the neural network has an accuracy method defined - test_acc=total_acc.numpy()/batches # convert the total accuracy value to numpy array and divide it by number of batches to get the average accuracy value for test data - except Exception: - pass - else: # if batch mode is not used (use all data at once) - output=self.nn.fp(test_data) # get the output value using the neural network's forward propagation method - test_loss=self.nn.loss(output,test_labels) # get the loss value using the neural network's loss function - test_loss=test_loss.numpy() # convert the loss value to numpy array - try: - test_acc=self.nn.accuracy(output,test_labels) # calculate the accuracy value using the neural network's accuracy method - test_acc=test_acc.numpy() # convert the accuracy value to numpy array - except Exception as e: - try: - if self.nn.accuracy!=None: # if the neural network has an accuracy method defined - raise e # raise the exception - except Exception: - pass - try: - if self.nn.accuracy!=None: # if the neural network has an accuracy method defined - return test_loss,test_acc # return the test loss and accuracy values - except Exception: - return test_loss,None # return the test loss value and None - - - def suspend_func(self): # a method to check whether a request to suspend the training or online learning has been made - if self.suspend==True: # if the suspend flag is set to True - if self.save_epoch==None: # if the save epoch number is None, it means no request to save the model has been made - print('Training have suspended.') # print a message to indicate the training or online learning has been suspended - else: - self._save() # save the model using the _save method - while True: # loop indefinitely until resumed by other conditions - if self.suspend==False: # if the suspend flag is set to False, it means a request to resume the training or online learning has been made - print('Training have continued.') # print a message to indicate the training or online learning has been resumed - break # break the loop and resume the training or online learning - return # return nothing for this case - - - def stop_func(self): # a method to check whether the training or online learning has reached the target loss or accuracy values and stop it accordingly - if self.end(): # check whether the target loss or accuracy values have been reached using the end method - self.save(self.total_epoch,True) # save the model using the save method with total epoch number and True flag as inputs - print('\nSystem have stopped training,Neural network have been saved.') # print a message to indicate the training or online learning has been stopped and the model has been saved - self._time=self.time-int(self.time) # get the fractional part of the time elapsed for this training session - if self._time<0.5: # if the fractional part is less than 0.5 - self.time=int(self.time) # round down the time elapsed to integer - else: # if the fractional part is greater than or equal to 0.5 - self.time=int(self.time)+1 # round up the time elapsed to integer - self.total_time+=self.time # calculate and update the total time elapsed for all trainings - print() # print an empty line for readability - print('epoch:{0}'.format(self.total_epoch)) # print the total epoch number for all trainings - if self.test_flag==False: # if test data is not used - print('last loss:{0:.6f}'.format(self.train_loss)) # print the last train loss value - else: # if test data is used - print('last loss:{0:.6f},last test loss:{1:.6f}'.format(self.train_loss,self.test_loss)) # print the last train loss value and test loss value - try: - if self.nn.accuracy!=None: # if the neural network has an accuracy method defined - if self.acc_flag=='%': # if percentage mode is used for accuracy display - if self.test_flag==False: # if test data is not used - print('last accuracy:{0:.1f}'.format(self.train_acc*100)) # print the last train accuracy value in percentage - else: # if test data is used - print('last accuracy:{0:.1f},last test accuracy:{1:.1f}'.format(self.train_acc*100,self.test_acc*100)) # print the last train accuracy value and test accuracy value in percentage - else: # if decimal mode is used for accuracy display - if self.test_flag==False: # if test data is not used - print('last accuracy:{0:.6f}'.format(self.train_acc)) # print the last train accuracy value in decimal - else: # if test data is used - print('last accuracy:{0:.6f},last test accuracy:{1:.6f}'.format(self.train_acc,self.test_acc)) # print the last train accuracy value and test accuracy value in decimal - except Exception: # if the neural network does not have an accuracy method defined or it fails - pass - print() # print an empty line for readability - print('time:{0}s'.format(self.total_time)) # print the total time elapsed for all trainings - self.stop_flag=True # set the stop flag to True, which means the training or online learning has been stopped - return True # return True to indicate that the training or online learning has been stopped - return False # return False to indicate that the training or online learning has not been stopped - - - def stop_func_(self): # a method to check whether a request to stop the training or online learning has been made or the target loss or accuracy values have been reached - if self.stop==True: # if the stop flag is set to True, it means a request to stop the training or online learning has been made - if self.stop_flag==True or self.stop_func(): # if the stop flag is already True, it means the target loss or accuracy values have been reached, or check whether the target loss or accuracy values have been reached using the stop_func method - return True # return True to indicate that the training or online learning has been stopped - return False # return False to indicate that the training or online learning has not been stopped - - - def visualize_train(self): # a method to visualize the train loss and accuracy values using matplotlib.pyplot - print() # print an empty line for readability - plt.figure(1) # create a new figure with number 1 - plt.plot(np.arange(self.total_epoch),self.train_loss_list) # plot the train loss list with x-axis as the epoch number - plt.title('train loss') # set the title of the plot to 'train loss' - plt.xlabel('epoch') # set the x-axis label to 'epoch' - plt.ylabel('loss') # set the y-axis label to 'loss' - print('train loss:{0:.6f}'.format(self.train_loss)) # print the last train loss value - try: - if self.nn.accuracy!=None: # if the neural network has an accuracy method defined - plt.figure(2) # create a new figure with number 2 - plt.plot(np.arange(self.total_epoch),self.train_acc_list) # plot the train accuracy list with x-axis as the epoch number - plt.title('train acc') # set the title of the plot to 'train acc' - plt.xlabel('epoch') # set the x-axis label to 'epoch' - plt.ylabel('acc') # set the y-axis label to 'acc' - if self.acc_flag=='%': # if percentage mode is used for accuracy display - print('train acc:{0:.1f}'.format(self.train_acc*100)) # print the last train accuracy value in percentage - else: # if decimal mode is used for accuracy display - print('train acc:{0:.6f}'.format(self.train_acc)) # print the last train accuracy value in decimal - except Exception: # if the neural network does not have an accuracy method defined or it fails - pass - return # return nothing for this case + # handle any exception raised in the previous block of code and assign it to e + if hasattr(self.nn,'accuracy'): + # check if the nn object has accuracy method + raise e + # re-raise e + else: + # check if the nn object does not have accuracy method + test_loss=parallel_test_.loss_acc() + # call loss_acc method of parallel_test_ object and assign the result to test_loss + elif batch!=None: + # check if batch argument is not None + total_loss=0 + # initialize total_loss to 0 + total_acc=0 + # initialize total_acc to 0 + if self.test_dataset!=None: + # check if test_dataset attribute is not None + batches=0 + # initialize batches to 0 + for data_batch,labels_batch in self.test_dataset: + # iterate over test_dataset attribute + batches+=1 + # add 1 to batches + if hasattr(self.platform,'DType'): + # check if the platform attribute has DType attribute, which indicates tensorflow platform + batch_loss,batch_acc=self.test_tf(data_batch,labels_batch) + # call test_tf method with data_batch and labels_batch as arguments and assign the results to batch_loss and batch_acc + else: + # check if the platform attribute does not have DType attribute, which indicates pytorch platform + batch_loss,batch_acc=self.test_pytorch(data_batch,labels_batch) + # call test_pytorch method with data_batch and labels_batch as arguments and assign the results to batch_loss and batch_acc + total_loss+=batch_loss # add batch_loss to total_loss + try: # try the following block of code + total_acc+=batch_acc # add batch_acc to total_acc + except Exception: # handle any exception raised in the previous block of code + pass # do nothing + test_loss=total_loss.numpy()/batches # convert total_loss to numpy array, divide it by batches, and assign it to test_loss + if hasattr(self.nn,'accuracy'): # check if the nn object has accuracy method + test_acc=total_acc.numpy()/batches # convert total_acc to numpy array, divide it by batches, and assign it to test_acc + else: # check if test_dataset attribute is None + shape0=test_data.shape[0] # get the first dimension of test_data and assign it to shape0 + batches=int((shape0-shape0%batch)/batch) # calculate the number of batches and assign it to batches + for j in range(batches): # iterate over batches + index1=j*batch # calculate the start index of a batch and assign it to index1 + index2=(j+1)*batch # calculate the end index of a batch and assign it to index2 + data_batch=test_data[index1:index2] # slice the test_data from index1 to index2 and assign it to data_batch + labels_batch=test_labels[index1:index2] # slice the test_labels from index1 to index2 and assign it to labels_batch + if hasattr(self.platform,'DType'): # check if the platform attribute has DType attribute, which indicates tensorflow platform + batch_loss,batch_acc=self.test_tf(data_batch,labels_batch) # call test_tf method with data_batch and labels_batch as arguments and assign the results to batch_loss and batch_acc + else: # check if the platform attribute does not have DType attribute, which indicates pytorch platform + batch_loss,batch_acc=self.test_pytorch(data_batch,labels_batch) # call test_pytorch method with data_batch and labels_batch as arguments and assign the results to batch_loss and batch_acc + total_loss+=batch_loss # add batch_loss to total_loss + if hasattr(self.nn,'accuracy'): # check if the nn object has accuracy method + total_acc+=batch_acc # add batch_acc to total_acc + if shape0%batch!=0: # check if there is a remainder after dividing shape0 by batch + batches+=1 # add 1 to batches + index1=batches*batch # calculate the start index of a batch and assign it to index1 + index2=batch-(shape0-batches*batch) # calculate the end index of a batch and assign it to index2 + try: # try the following block of code + try: # try the following block of code + data_batch=self.platform.concat([test_data[index1:],test_data[:index2]],0) + # concatenate the test_data from index1 to the end and from the beginning to index2 along the first axis and assign it to data_batch + labels_batch=self.platform.concat([test_labels[index1:],test_labels[:index2]],0) + # concatenate the test_labels from index1 to the end and from the beginning to index2 along the first axis and assign it to labels_batch + except Exception: # handle any exception raised in the previous block of code + data_batch=np.concatenate([test_data[index1:],test_data[:index2]],0) + # concatenate the test_data from index1 to the end and from the beginning to index2 along the first axis using numpy and assign it to data_batch + labels_batch=np.concatenate([test_labels[index1:],test_labels[:index2]],0) + # concatenate the test_labels from index1 to the end and from the beginning to index2 along the first axis using numpy and assign it to labels_batch + except Exception as e: # handle any exception raised in the previous block of code and assign it to e + raise e # re-raise e + if hasattr(self.platform,'DType'): # check if the platform attribute has DType attribute, which indicates tensorflow platform + batch_loss,batch_acc=self.test_tf(data_batch,labels_batch) # call test_tf method with data_batch and labels_batch as arguments and assign the results to batch_loss and batch_acc + else: # check if the platform attribute does not have DType attribute, which indicates pytorch platform + batch_loss,batch_acc=self.test_pytorch(data_batch,labels_batch) # call test_pytorch method with data_batch and labels_batch as arguments and assign the results to batch_loss and batch_acc + total_loss+=batch_loss # add batch_loss to total_loss + if hasattr(self.nn,'accuracy'): # check if the nn object has accuracy method + total_acc+=batch_acc # add batch_acc to total_acc + if hasattr(self.platform,'DType'): # check if the platform attribute has DType attribute, which indicates tensorflow platform + test_loss=total_loss.numpy()/batches # convert total_loss to numpy array, divide it by batches, and assign it to test_loss + else: # check if the platform attribute does not have DType attribute, which indicates pytorch platform + test_loss=total_loss.detach().numpy()/batches # detach total_loss from computation graph, convert it to numpy array, divide it by batches, and assign it to test_loss + if hasattr(self.nn,'accuracy'): # check if the nn object has accuracy method + test_acc=total_acc.numpy()/batches # convert total_acc to numpy array, divide it by batches, and assign it to test_acc + else: # check if batch argument is None + if hasattr(self.platform,'DType'): # check if the platform attribute has DType attribute, which indicates tensorflow platform + batch_loss,batch_acc=self.test_tf(test_data,test_labels) # call test_tf method with test_data and test_labels as arguments and assign the results to batch_loss and batch_acc + else: # check if the platform attribute does not have DType attribute, which indicates pytorch platform + batch_loss,batch_acc=self.test_pytorch(test_data,test_labels) # call test_pytorch method with test_data and test_labels as arguments and assign the results to batch_loss and batch_acc + test_loss=batch_loss.numpy() # convert batch_loss to numpy array and assign it to test_loss + if hasattr(self.nn,'accuracy'): # check if the nn object has accuracy method + test_acc=batch_acc.numpy() # convert batch_acc to numpy array and assign it to test_acc + if hasattr(self.nn,'accuracy'): # check if the nn object has accuracy method + return test_loss,test_acc # return test_loss and test_acc + else: # check if the nn object does not have accuracy method + return test_loss # return test_loss From 8a5f9f872fc6c98fc06c8fa2bb59e86a9315fad5 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Sat, 22 Jul 2023 21:37:55 +0800 Subject: [PATCH 072/337] Update kernel.py --- .../RL/kernel/nspn/kernel.py | 1774 +++++++++++------ 1 file changed, 1131 insertions(+), 643 deletions(-) diff --git a/Note 7.0 documentation/RL/kernel/nspn/kernel.py b/Note 7.0 documentation/RL/kernel/nspn/kernel.py index 0ddd98a9..f7ccf658 100644 --- a/Note 7.0 documentation/RL/kernel/nspn/kernel.py +++ b/Note 7.0 documentation/RL/kernel/nspn/kernel.py @@ -1,685 +1,1173 @@ +# import the necessary modules from tensorflow import function from tensorflow import data as tf_data import numpy as np -import matplotlib.pyplot as plt import statistics import time - +# define a class named kernel class kernel: + # define the initialization method def __init__(self,nn=None,save_episode=False): - self.nn=nn # the neural network model to be trained - try: - self.nn.km=1 # a flag to indicate that the model is using kernel method - except Exception: - pass - self.platform=None # the platform to use, such as TensorFlow or PyTorch - self.state_pool=None # the pool of states - self.action_pool=None # the pool of actions - self.next_state_pool=None # the pool of next states - self.reward_pool=None # the pool of rewards - self.done_pool=None # the pool of done flags - self.episode_set=[] # the list of episodes - self.epsilon=None # the epsilon value for epsilon-greedy policy - self.episode_step=None # the maximum number of steps per episode - self.pool_size=None # the maximum size of the pool - self.batch=None # the batch size for training - self.update_step=None # the frequency of updating the network parameters - self.trial_count=None # the number of trials to calculate the average reward - self.criterion=None # the criterion for stopping the training - self.reward_list=[] # the list of rewards per episode - self.max_episode_count=None # the maximum number of episodes to save - self.save_episode=save_episode # a flag to indicate whether to save episodes or not - self.loss=None # the loss value for training - self.loss_list=[] # the list of losses per episode - self.sc=0 # a counter for steps - self.total_episode=0 # a counter for episodes - self.time=0 # a timer for training time - self.total_time=0 - - - def action_vec(self): - if self.epsilon!=None: - self.action_one=np.ones(self.action_count,dtype=np.int8) # a vector of ones for action probabilities - return - - - def init(self): - try: - self.nn.pr.TD=np.array(0) # initialize the TD error for prioritized replay buffer - except Exception as e: - try: - if self.nn.pr!=None: - raise e - except Exception: - pass + # assign the nn parameter to the self.nn attribute + self.nn=nn + # if the nn attribute has a km attribute, assign 1 to it + if hasattr(self.nn,'km'): + self.nn.km=1 + # initialize the platform attribute as None + self.platform=None + # initialize the state_pool, action_pool, next_state_pool, reward_pool, done_pool attributes as None + self.state_pool=None + self.action_pool=None + self.next_state_pool=None + self.reward_pool=None + self.done_pool=None + # initialize the episode_set attribute as an empty list self.episode_set=[] - self.state_pool=None - self.action_pool=None - self.next_state_pool=None - self.reward_pool=None - self.done_pool=None + # initialize the epsilon attribute as None + self.epsilon=None + # initialize the episode_step attribute as None + self.episode_step=None + # initialize the pool_size attribute as None + self.pool_size=None + # initialize the batch attribute as None + self.batch=None + # initialize the update_step attribute as None + self.update_step=None + # initialize the trial_count attribute as None + self.trial_count=None + # initialize the criterion attribute as None + self.criterion=None + # initialize the reward_list attribute as an empty list self.reward_list=[] - self.loss=0 + # initialize the suspend attribute as False + self.suspend=False + # initialize the save_epi attribute as None + self.save_epi=None + # initialize the max_episode_count attribute as None + self.max_episode_count=None + # assign the save_episode parameter to the save_episode attribute + self.save_episode=save_episode + # assign the filename parameter to the filename attribute + self.filename='save.dat' + # initialize the loss attribute as 0 + self.loss=0 + # initialize the loss_list attribute as an empty list self.loss_list=[] - self.sc=0 - self.total_episode=0 - self.time=0 - self.total_time=0 - return - - - def set_up(self,epsilon=None,episode_step=None,pool_size=None,batch=None,update_step=None,trial_count=None,criterion=None): - if epsilon!=None: - self.epsilon=epsilon # set the epsilon value for epsilon-greedy policy - if episode_step!=None: - self.episode_step=episode_step # set the maximum number of steps per episode - if pool_size!=None: - self.pool_size=pool_size # set the maximum size of the pool - if batch!=None: - self.batch=batch # set the batch size for training - if update_step!=None: - self.update_step=update_step # set the frequency of updating the network parameters - if trial_count!=None: - self.trial_count=trial_count # set the number of trials to calculate the average reward - if criterion!=None: - self.criterion=criterion # set the criterion for stopping the training - self.action_vec() # create an action vector with the same length as the number of actions, and fill it with ones to indicate the initial probabilities of each action - return - - - def epsilon_greedy_policy(self,s): - action_prob=self.action_one*self.epsilon/len(self.action_one) # initialize the action probabilities with epsilon - try: - best_a=np.argmax(self.nn.nn.fp(s)) # find the best action according to the network output - action_prob[best_a]+=1-self.epsilon # increase the probability of the best action by 1-epsilon - except Exception as e: + # initialize the sc attribute as 0 + self.sc=0 + # initialize the total_episode attribute as 0 + self.total_episode=0 + # initialize the time attribute as 0 + self.time=0 + # initialize the total_time attribute as 0 + self.total_time=0 + + # define a method named action_vec + def action_vec(self): + # if the epsilon attribute is not None, assign a numpy array of ones with the length of action_count and multiply it by epsilon divided by the length of action_one to the action_one attribute + if self.epsilon!=None: + self.action_one=np.ones(self.action_count,dtype=np.int8) + # return nothing + return + + # define a method named init + def init(self): + # try to execute the following statements try: - if self.platform.DType!=None: # check if the platform is TensorFlow - raise e - except Exception: - best_a=self.nn.nn(s).argmax() # find the best action according to the network output - action_prob[best_a.numpy()]+=1-self.epsilon # increase the probability of the best action by 1-epsilon - return action_prob # return the action probabilities - - - @function(jit_compile=True) - def tf_opt(self,state_batch,action_batch,next_state_batch,reward_batch,done_batch): - with self.platform.GradientTape(persistent=True) as tape: # create a persistent gradient tape to record the gradients - loss=self.nn.loss(state_batch,action_batch,next_state_batch,reward_batch,done_batch) # calculate the loss function - try: + # if the nn attribute has a pr attribute, assign a numpy array of 0 to its TD attribute + if hasattr(self.nn,'pr'): + self.nn.pr.TD=np.array(0) + # if an exception occurs, raise it + except Exception as e: + raise e + # assign False to the suspend attribute + self.suspend=False + # assign None to the save_epi attribute + self.save_epi=None + # assign an empty list to the episode_set attribute + self.episode_set=[] + # assign None to the state_pool, action_pool, next_state_pool, reward_pool, done_pool attributes + self.state_pool=None + self.action_pool=None + self.next_state_pool=None + self.reward_pool=None + self.done_pool=None + # assign an empty list to the reward_list attribute + self.reward_list=[] + # assign 0 to the loss attribute + self.loss=0 + # assign an empty list to the loss_list attribute + self.loss_list=[] + # assign 0 to the sc attribute + self.sc=0 + # assign 0 to the total_episode attribute + self.total_episode=0 + # assign 0 to the time attribute + self.time=0 + # assign 0 to the total_time attribute + self.total_time=0 + + # define a method named set_up + def set_up(self,epsilon=None,episode_step=None,pool_size=None,batch=None,update_step=None,trial_count=None,criterion=None): + # if epsilon is not None, assign it to the epsilon attribute + if epsilon!=None: + self.epsilon=epsilon + # if episode_step is not None, assign it to the episode_step attribute + if episode_step!=None: + self.episode_step=episode_step + # if pool_size is not None, assign it to the pool_size attribute + if pool_size!=None: + self.pool_size=pool_size + # if batch is not None, assign it to the batch attribute + if batch!=None: + self.batch=batch + # if update_step is not None, assign it to the update_step attribute + if update_step!=None: + self.update_step=update_step + # if trial_count is not None, assign it to the trial_count attribute + if trial_count!=None: + self.trial_count=trial_count + # if criterion is not None, assign it to the criterion attribute + if criterion!=None: + self.criterion=criterion + # call the action_vec method + self.action_vec() + # return nothing + return + + # define a method named epsilon_greedy_policy + def epsilon_greedy_policy(self,s): + # assign a numpy array of ones with the length of action_count and multiply it by epsilon divided by the length of action_one to the action_prob variable + action_prob=self.action_one*self.epsilon/len(self.action_one) + # try to execute the following statements try: - gradient=self.nn.gradient(tape,loss) # calculate the gradient using the custom gradient function - try: - self.nn.opt.apply_gradients(zip(gradient,self.nn.param)) # apply the gradient to update the network parameters - except Exception: - self.nn.opt(gradient) # apply the gradient to update the network parameters - except Exception: - try: - if self.nn.nn!=None: # check if the network is a single network or a pair of networks - gradient=tape.gradient(loss,self.nn.param) # calculate the gradient using the tape function - self.nn.opt.apply_gradients(zip(gradient,self.nn.param)) # apply the gradient to update the network parameters - except Exception: - actor_gradient=tape.gradient(loss[0],self.nn.param[0]) # calculate the gradient for the actor network - critic_gradient=tape.gradient(loss[1],self.nn.param[1]) # calculate the gradient for the critic network - self.nn.opt.apply_gradients(zip(actor_gradient,self.nn.param[0])) # apply the gradient to update the actor network parameters - self.nn.opt.apply_gradients(zip(critic_gradient,self.nn.param[1])) # apply the gradient to update the critic network parameters - except Exception as e: - raise e - return loss # return the loss value - - - def pytorch_opt(self,state_batch,action_batch,next_state_batch,reward_batch,done_batch): - loss=self.nn.loss(state_batch,action_batch,next_state_batch,reward_batch,done_batch) # calculate the loss function - self.nn.backward(loss) # calculate and accumulate the gradients using the custom backward function - self.nn.opt() # apply the gradients to update the network parameters using the custom opt function - return loss # return the loss value - - - def opt(self,state_batch,action_batch,next_state_batch,reward_batch,done_batch): - try: - if self.platform.DType!=None: # check if the platform is TensorFlow - loss=self.tf_opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch) # use TensorFlow optimization method - except Exception: - loss=self.pytorch_opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch) # use PyTorch optimization method - return loss # return the loss value - - - def pool(self,s,a,next_s,r,done): - if type(self.state_pool)!=np.ndarray and self.state_pool==None: # check if the pool is empty or not - self.state_pool=s # initialize the state pool with s - if type(a)==int: # check if a is an integer or a vector - if type(self.nn.param[0])!=list: - a=np.array(a,self.nn.param[0].dtype.name) # convert a to an array with dtype matching nn.param[0] + # assign the index of the maximum value of the output of the nn attribute's fp method with s as input to the best_a variable + best_a=np.argmax(self.nn.nn.fp(s)) + # add 1 minus epsilon to the best_a element of action_prob + action_prob[best_a]+=1-self.epsilon + # if an exception occurs, execute the following statements + except Exception as e: + # if the platform attribute has a DType attribute, raise the exception + if hasattr(self.platform,'DType'): + raise e + # else, assign the index of the maximum value of the output of the nn attribute's nn method with s as input to the best_a variable else: - a=np.array(a,self.nn.param[0][0].dtype.name) # convert a to an array with dtype matching nn.param[0][0] - self.action_pool=np.expand_dims(a,axis=0) # initialize the action pool with a and add an extra dimension for batch size + best_a=self.nn.nn(s).argmax() + # add 1 minus epsilon to the best_a element of action_prob and convert it to numpy array + action_prob[best_a.numpy()]+=1-self.epsilon + # return action_prob + return action_prob + + # define a method named get_reward + def get_reward(self,max_step=None,seed=None): + # initialize the reward variable as 0 + reward=0 + # if seed is None, assign the output of the genv attribute's reset method to the s variable + if seed==None: + s=self.genv.reset() + # else, assign the output of the genv attribute's reset method with seed as input to the s variable else: - if type(self.nn.param[0])!=list: - a=a.astype(self.nn.param[0].dtype.name) # convert a to an array with dtype matching nn.param[0] + s=self.genv.reset(seed=seed) + # if max_step is not None, execute the following statements + if max_step!=None: + # use a for loop to iterate from 0 to max_step + for i in range(max_step): + # if the end_flag attribute is True, break the loop + if self.end_flag==True: + break + # if the nn attribute has a nn attribute, execute the following statements + if hasattr(self.nn,'nn'): + # if the platform attribute has a DType attribute, execute the following statements + if hasattr(self.platform,'DType'): + # expand the dimension of s along axis 0 and assign it back to s + s=np.expand_dims(s,axis=0) + # assign the index of the maximum value of the output of the nn attribute's nn method with s as input to the a variable + a=np.argmax(self.nn.nn.fp(s)) + # else, execute the following statements + else: + # expand the dimension of s along axis 0 and assign it back to s + s=np.expand_dims(s,axis=0) + # convert s to a tensor with float dtype and assign it to the device attribute of the nn attribute and assign it back to s + s=self.platform.tensor(s,dtype=self.platform.float).to(self.nn.device) + # assign the index of the maximum value of the output of the nn attribute's nn method with s as input to the a variable and convert it to numpy array + a=self.nn.nn(s).detach().numpy().argmax() + # else, execute the following statements + else: + # if the nn attribute has an action attribute, execute the following statements + if hasattr(self.nn,'action'): + # if the platform attribute has a DType attribute, execute the following statements + if hasattr(self.platform,'DType'): + # expand the dimension of s along axis 0 and assign it back to s + s=np.expand_dims(s,axis=0) + # assign the output of the nn attribute's action method with s as input to the a variable and convert it to numpy array + a=self.nn.action(s).numpy() + # else, execute the following statements + else: + # expand the dimension of s along axis 0 and assign it back to s + s=np.expand_dims(s,axis=0) + # convert s to a tensor with float dtype and assign it to the device attribute of the nn attribute and assign it back to s + s=self.platform.tensor(s,dtype=self.platform.float).to(self.nn.device) + # assign the output of the nn attribute's action method with s as input to the a variable and convert it to numpy array and detach it from computation graph + a=self.nn.action(s).detach().numpy() + # else, execute the following statements + else: + # if the platform attribute has a DType attribute, execute the following statements + if hasattr(self.platform,'DType'): + # expand the dimension of s along axis 0 and assign it back to s + s=np.expand_dims(s,axis=0) + # assign the output of adding up output of fp method from actor attribute of nn with noise from nn as input to a variable and convert it to numpy array + a=(self.nn.actor.fp(s)+self.nn.noise()).numpy() + # squeeze out any singleton dimensions from a and assign it back to a + a=np.squeeze(a) + # else, execute the following statements + else: + # expand the dimension of s along axis 0 and assign it back to s + s=np.expand_dims(s,axis=0) + # convert s to a tensor with float dtype and assign it to device attribute of nn and assign it back to s + s=self.platform.tensor(s,dtype=self.platform.float).to(self.nn.device) + # assign output of adding up output from actor method from nn with noise from nn as input to a variable and convert it to numpy array and detach it from computation graph + a=(self.nn.actor(s)+self.nn.noise()).detach().numpy() + # squeeze out any singleton dimensions from a and assign it back to a + a=np.squeeze(a) + # assign the output of the genv attribute's step method with a as input to the next_s, r, done variables + next_s,r,done,_=self.genv.step(a) + # assign next_s to s + s=next_s + # add r to reward and assign it back to reward + reward+=r + # if the nn attribute has a stop attribute, execute the following statements + if hasattr(self.nn,'stop'): + # if the output of the nn attribute's stop method with next_s as input is True, break the loop + if self.nn.stop(next_s): + break + # if done is True, break the loop + if done: + break + # return reward + return reward + # else, execute the following statements + else: + # use a while loop to iterate indefinitely + while True: + # if the end_flag attribute is True, break the loop + if self.end_flag==True: + break + # if the nn attribute has a nn attribute, execute the following statements + if hasattr(self.nn,'nn'): + # if the platform attribute has a DType attribute, execute the following statements + if hasattr(self.platform,'DType'): + # expand the dimension of s along axis 0 and assign it back to s + s=np.expand_dims(s,axis=0) + # assign the index of the maximum value of the output of the nn attribute's nn method with s as input to the a variable + a=np.argmax(self.nn.nn.fp(s)) + # else, execute the following statements + else: + # expand the dimension of s along axis 0 and assign it back to s + s=np.expand_dims(s,axis=0) + # convert s to a tensor with float dtype and assign it to the device attribute of the nn attribute and assign it back to s + s=self.platform.tensor(s,dtype=self.platform.float).to(self.nn.device) + # assign the index of the maximum value of the output of the nn attribute's nn method with s as input to the a variable and convert it to numpy array + a=self.nn.nn(s).detach().numpy().argmax() + # else, execute the following statements + else: + # if the nn attribute has an action attribute, execute the following statements + if hasattr(self.nn,'action'): + # if the platform attribute has a DType attribute, execute the following statements + if hasattr(self.platform,'DType'): + # expand the dimension of s along axis 0 and assign it back to s + s=np.expand_dims(s,axis=0) + # assign the output of the nn attribute's action method with s as input to the a variable and convert it to numpy array + a=self.nn.action(s).numpy() + # else, execute the following statements + else: + # expand the dimension of s along axis 0 and assign it back to s + s=np.expand_dims(s,axis=0) + # convert s to a tensor with float dtype and assign it to device attribute of nn and assign it back to s + s=self.platform.tensor(s,dtype=self.platform.float).to(self.nn.device) + # assign output of action method from nn with s as input to a variable and convert it to numpy array and detach it from computation graph + a=self.nn.action(s).detach().numpy() + # else, execute the following statements + else: + # if platform has DType attribute, execute following statements + if hasattr(self.platform,'DType'): + # expand dimension of s along axis 0 and assign it back to s + s=np.expand_dims(s,axis=0) + # assign output of adding up output from fp method from actor attribute of nn with noise from nn as input to a variable and convert it to numpy array + a=(self.nn.actor.fp(s)+self.nn.noise()).numpy() + # squeeze out any singleton dimensions from a and assign it back to a + a=np.squeeze(a) + # else, execute following statements + else: + # expand dimension of s along axis 0 and assign it back to s + s=np.expand_dims(s,axis=0) + # convert s to tensor with float dtype and assign it to device attribute of nn and assign it back to s + s=self.platform.tensor(s,dtype=self.platform.float).to(self.nn.device) + # assign output of adding up output from actor method from nn with noise from nn as input to a variable and convert it to numpy array and detach it from computation graph + a=(self.nn.actor(s)+self.nn.noise()).detach().numpy() + # squeeze out any singleton dimensions from a and assign it back to a + a=np.squeeze(a) + # assign output of genv attribute's step method with a as input to next_s, r, done variables + next_s,r,done,_=self.genv.step(a) + # assign next_s to s + s=next_s + # add r to reward and assign it back to reward + reward+=r + # if the nn attribute has a stop attribute, execute the following statements + if hasattr(self.nn,'stop'): + # if the output of the nn attribute's stop method with next_s as input is True, break the loop + if self.nn.stop(next_s): + break + # if done is True, break the loop + if done: + break + # return reward + return reward + + # define a method named get_episode + def get_episode(self,max_step=None,seed=None): + # initialize the counter variable as 0 + counter=0 + # initialize the episode variable as an empty list + episode=[] + # if seed is None, assign the output of the genv attribute's reset method to the s variable + if seed==None: + s=self.genv.reset() + # else, assign the output of the genv attribute's reset method with seed as input to the s variable + else: + s=self.genv.reset(seed=seed) + # initialize the end_flag attribute as False + self.end_flag=False + # use a while loop to iterate indefinitely + while True: + # if the end_flag attribute is True, break the loop + if self.end_flag==True: + break + # if the nn attribute has a nn attribute, execute the following statements + if hasattr(self.nn,'nn'): + # if the platform attribute has a DType attribute, execute the following statements + if hasattr(self.platform,'DType'): + # expand the dimension of s along axis 0 and assign it back to s + s=np.expand_dims(s,axis=0) + # assign the index of the maximum value of the output of the nn attribute's nn method with s as input to the a variable + a=np.argmax(self.nn.nn.fp(s)) + # else, execute the following statements + else: + # expand the dimension of s along axis 0 and assign it back to s + s=np.expand_dims(s,axis=0) + # convert s to a tensor with float dtype and assign it to the device attribute of the nn attribute and assign it back to s + s=self.platform.tensor(s,dtype=self.platform.float).to(self.nn.device) + # assign the index of the maximum value of the output of the nn attribute's nn method with s as input to the a variable and convert it to numpy array + a=self.nn.nn(s).detach().numpy().argmax() + # else, execute the following statements else: - a=a.astype(self.nn.param[0][0].dtype.name) # convert a to an array with dtype matching nn.param[0][0] - self.action_poolself.action_pool=a # initialize the action pool with a - self.next_state_pool=np.expand_dims(next_s,axis=0) # initialize the next state pool with next_s and add an extra dimension for batch size - self.reward_pool=np.expand_dims(r,axis=0) # initialize the reward pool with r and add an extra dimension for batch size - self.done_pool=np.expand_dims(done,axis=0) # initialize the done pool with done and add an extra dimension for batch size - else: - self.state_pool=np.concatenate((self.state_pool,s),0) # append s to the state pool along the first axis - if type(a)==int: # check if a is an integer or a vector - if type(self.nn.param[0])!=list: - a=np.array(a,self.nn.param[0].dtype.name) # convert a to an array with dtype matching nn.param[0] + # if the nn attribute has an action attribute, execute the following statements + if hasattr(self.nn,'action'): + # if the platform attribute has a DType attribute, execute the following statements + if hasattr(self.platform,'DType'): + # expand the dimension of s along axis 0 and assign it back to s + s=np.expand_dims(s,axis=0) + # assign output of action method from nn with s as input to a variable and convert it to numpy array + a=self.nn.action(s).numpy() + # else, execute following statements + else: + # expand dimension of s along axis 0 and assign it back to s + s=np.expand_dims(s,axis=0) + # convert s to tensor with float dtype and assign it to device attribute of nn and assign it back to s + s=self.platform.tensor(s,dtype=self.platform.float).to(self.nn.device) + # assign output of action method from nn with s as input to a variable and convert it to numpy array and detach it from computation graph + a=self.nn.action(s).detach().numpy() + # else, execute following statements + else: + # if platform has DType attribute, execute following statements + if hasattr(self.platform,'DType'): + # expand dimension of s along axis 0 and assign it back to s + s=np.expand_dims(s,axis=0) + # assign output of adding up output from fp method from actor attribute of nn with noise from nn as input to a variable and convert it to numpy array + a=(self.nn.actor.fp(s)+self.nn.noise()).numpy() + # squeeze out any singleton dimensions from a and assign it back to a + a=np.squeeze(a) + # else, execute following statements + else: + # expand dimension of s along axis 0 and assign it back to s + s=np.expand_dims(s,axis=0) + # convert s to tensor with float dtype and assign it to device attribute of nn and assign it back to s + s=self.platform.tensor(s,dtype=self.platform.float).to(self.nn.device) + # assign output of adding up output from actor method from nn with noise from nn as input to a variable and convert it to numpy array and detach it from computation graph + a=(self.nn.actor(s)+self.nn.noise()).detach().numpy() + # squeeze out any singleton dimensions from a and assign it back to a + a=np.squeeze(a) + # assign the output of the genv attribute's step method with a as input to the next_s, r, done variables + next_s,r,done,_=self.genv.step(a) + # if the end_flag attribute is True, break the loop + if self.end_flag==True: + break + # if done is True, execute the following statements + if done: + # append a list of s, a, next_s, r to the episode variable + episode.append([s,a,next_s,r]) + # append the string 'done' to the episode variable + episode.append('done') + # break the loop + break + # else, execute the following statements + else: + # append a list of s, a, next_s, r to the episode variable + episode.append([s,a,next_s,r]) + # if max_step is not None and counter equals max_step minus 1, break the loop + if max_step!=None and counter==max_step-1: + break + # assign next_s to s + s=next_s + # increment counter by 1 + counter+=1 + # return episode + return episode + + # define a method named tf_opt + @function(jit_compile=True) + def tf_opt(self,state_batch,action_batch,next_state_batch,reward_batch,done_batch): + # use the platform attribute's GradientTape method with persistent as True as a context manager and assign it to the tape variable + with self.platform.GradientTape(persistent=True) as tape: + # assign the output of the nn attribute's loss method with state_batch, action_batch, next_state_batch, reward_batch, done_batch as inputs to the loss variable + loss=self.nn.loss(state_batch,action_batch,next_state_batch,reward_batch,done_batch) + # if the nn attribute has a gradient attribute, execute the following statements + if hasattr(self.nn,'gradient'): + # assign the output of the nn attribute's gradient method with tape and loss as inputs to the gradient variable + gradient=self.nn.gradient(tape,loss) + # if the opt attribute of the nn attribute has an apply_gradients method, execute the following statements + if hasattr(self.nn.opt,'apply_gradients'): + # call the apply_gradients method of the opt attribute of the nn attribute with zip function of gradient and param attribute of nn as input + self.nn.opt.apply_gradients(zip(gradient,self.nn.param)) + # else, execute following statements else: - a=np.array(a,self.nn.param[0][0].dtype.name) # convert a to an array with dtype matching nn.param[0][0] - self.action_pool=np.concatenate((self.action_pool,np.expand_dims(a,axis=0)),0) # append a to the action pool along the first axis and add an extra dimension for batch size + # call opt attribute of nn with gradient as input + self.nn.opt(gradient) + # else, execute following statements else: - if type(self.nn.param[0])!=list: - a=a.astype(self.nn.param[0].dtype.name) # convert a to an array with dtype matching nn.param[0] + # if nn has nn attribute, execute following statements + if hasattr(self.nn,'nn'): + # assign output of tape's gradient method with loss and param attribute of nn as inputs to gradient variable + gradient=tape.gradient(loss,self.nn.param) + # call apply_gradients method of opt attribute of nn with zip function of gradient and param attribute of nn as input + self.nn.opt.apply_gradients(zip(gradient,self.nn.param)) + # else, execute following statements else: - a=a.astype(self.nn.param[0][0].dtype.name) # convert a to an array with dtype matching nn.param[0][0] - self.action_pool=np.concatenate((self.action_pool,a),0) # append a to the action pool along the first axis - self.next_state_pool=np.concatenate((self.next_state_pool,np.expand_dims(next_s,axis=0)),0) # append next_s to the next state pool along the first axis and add an extra dimension for batch size - self.reward_pool=np.concatenate((self.reward_pool,np.expand_dims(r,axis=0)),0) # append r to the reward pool along the first axis and add an extra dimension for batch size - self.done_pool=np.concatenate((self.done_pool,np.expand_dims(done,axis=0)),0) # append done to the done pool along the first axis and add an extra dimension for batch size - if len(self.state_pool)>self.pool_size: # check if the pool size exceeds the maximum size - self.state_pool=self.state_pool[1:] # remove the oldest state from the state pool - self.action_pool=self.action_pool[1:] # remove the oldest action from the action pool - self.next_state_pool=self.next_state_pool[1:] # remove the oldest next state from the next state pool - self.reward_pool=self.reward_pool[1:] # remove the oldest reward from the reward pool - self.done_pool=self.done_pool[1:] # remove the oldest done flag from the done pool - return - - - def _train(self): - if len(self.state_pool)self.pool_size: # check if the pool size exceeds the maximum size - TD=np.array(0) # create a zero TD error value - self.nn.pr.TD=np.append(TD,self.nn.pr.TD[2:]) # remove the oldest TD error value from the TD error list and append the zero TD error value - except Exception as e: - try: - if self.nn.pr!=None: # check if the prioritized replay buffer is defined or not - raise e # raise the exception if it is defined - except Exception: - pass - self.reward=r+self.reward # accumulate the reward value over steps - loss=self._train() # train the network using the pool data and get the loss value - self.sc+=1 # increase the step counter by one - if done: # check if done flag is True or not - if self.save_episode==True: # check if save episode flag is True or not - episode=[s,a,next_s,r] # create a list of data for this step - self.reward_list.append(self.reward) # append the reward value to the reward list - return loss,episode,done # return the loss value, episode data, and done flag - elif self.save_episode==True: # check if save episode flag is True or not - episode=[s,a,next_s,r] # create a list of data for this step - s=next_s # update s as next_s for next step - else: - for _ in range(self.episode_step): # loop over the maximum number of steps per episode - try: - try: - if self.platform.DType!=None: # check if the platform is TensorFlow - s=np.expand_dims(s,axis=0) # add an extra dimension for batch size to s - if self.epsilon==None: # check if the epsilon value is defined or not - self.epsilon=self.nn.epsilon(self.sc) # calculate the epsilon value using the custom epsilon function - action_prob=self.epsilon_greedy_policy(s) # get the action probabilities using the epsilon-greedy policy function - a=np.random.choice(self.action_count,p=action_prob) # choose an action randomly according to the action probabilities - except Exception: - s=np.expand_dims(s,axis=0) # add an extra dimension for batch size to s - s=self.platform.tensor(s,dtype=self.platform.float).to(self.nn.device) # convert s to a tensor with dtype matching platform.float and move it to the device - if self.epsilon==None: # check if the epsilon value is defined or not - self.epsilon=self.nn.epsilon(self.sc) # calculate the epsilon value using the custom epsilon function - action_prob=self.epsilon_greedy_policy(s) # get the action probabilities using the epsilon-greedy policy function - a=np.random.choice(self.action_count,p=action_prob) # choose an action randomly according to the action probabilities - next_s,r,done=self.nn.env(a) # get the next state, reward, and done flag from the environment using the custom env function and the chosen action - try: - if self.platform.DType!=None: # check if the platform is TensorFlow - if type(self.nn.param[0])!=list: - next_s=np.array(next_s,self.nn.param[0].dtype.name) # convert next_s to an array with dtype matching nn.param[0] - r=np.array(r,self.nn.param[0].dtype.name) # convert r to an array with dtype matching nn.param[0] - done=np.array(done,self.nn.param[0].dtype.name) # convert done to an array with dtype matching nn.param[0] - else: - next_s=np.array(next_s,self.nn.param[0][0].dtype.name) # convert next_s to an array with dtype matching nn.param[0][0] - r=np.array(r,self.nn.param[0][0].dtype.name) # convert r to an array with dtype matching nn.param[0][0] - done=np.array(done,self.nn.param[0][0].dtype.name) # convert done to an array with dtype matching nn.param[0][0] - except Exception: - pass - except Exception as e: - try: - if self.nn.nn!=None: # check if the network is a single network or a pair of networks - raise e # raise the exception if it is a single network - except Exception: - try: + # initialize j variable as 0 + j=0 + # assign output from Dataset method from tf_data module with tensor_slices method with state_pool, action_pool, next_state_pool, reward_pool, done_pool as input and shuffle method with length of state_pool as input and batch method with batch as input to train_ds variable + train_ds=tf_data.Dataset.from_tensor_slices((self.state_pool,self.action_pool,self.next_state_pool,self.reward_pool,self.done_pool)).shuffle(len(self.state_pool)).batch(self.batch) + # use for loop to iterate over train_ds and assign outputs to state_batch, action_batch, next_state_batch, reward_batch, done_batch variables + for state_batch,action_batch,next_state_batch,reward_batch,done_batch in train_ds: + # call suspend_func method + self.suspend_func() + # assign output from opt method with state_batch, action_batch, next_state_batch, reward_batch, done_batch as inputs to batch_loss variable + batch_loss=self.opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch) + # add batch_loss to loss and assign it back to loss + loss+=batch_loss + # if nn has bc attribute, execute following statements + if hasattr(self.nn,'bc'): + # try to execute following statements try: - if self.nn.action!=None: # check if the custom action function is defined or not - try: - if self.platform.DType!=None: # check if the platform is TensorFlow - s=np.expand_dims(s,axis=0) # add an extra dimension for batch size to s - if self.epsilon==None: # check if the epsilon value is defined or not - self.epsilon=self.nn.epsilon(self.sc) # calculate the epsilon value using the custom epsilon function - try: - if self.nn.discriminator!=None: # check if the custom discriminator function is defined or not - a=self.nn.action(s) # get the action using the custom action function - reward=self.nn.discriminator(s,a) # get the reward using the custom discriminator function - s=np.squeeze(s) # remove the extra dimension from s - except Exception: - a=self.nn.action(s).numpy() # get the action using the custom action function and convert it to numpy array - except Exception: - s=np.expand_dims(s,axis=0) # add an extra dimension for batch size to s - s=self.platform.tensor(s,dtype=self.platform.float).to(self.nn.device) # convert s to a tensor with dtype matching platform.float and move it to the device - if self.epsilon==None: # check if the epsilon value is defined or not - self.epsilon=self.nn.epsilon(self.sc) # calculate the epsilon value using the custom epsilon function - try: - if self.nn.discriminator!=None: # check if the custom discriminator function is defined or not - a=self.nn.action(s) # get the action using thecustom action function - reward=self.nn.discriminator(s,a) # get the reward using the custom discriminator function - s=np.squeeze(s) # remove the extra dimension from s - except Exception: - a=self.nn.action(s).detach().numpy() # get the action using the custom action function and convert it to numpy array + # call assign_add method of bc attribute of nn with 1 as input + self.nn.bc.assign_add(1) + # if exception occurs, execute following statements except Exception: - try: - if self.platform.DType!=None: # check if the platform is TensorFlow - s=np.expand_dims(s,axis=0) # add an extra dimension for batch size to s - a=(self.nn.actor.fp(s)+self.nn.noise()).numpy() # get the action using the actor network and add some noise and convert it to numpy array - except Exception: - s=np.expand_dims(s,axis=0) # add an extra dimension for batch size to s - s=self.platform.tensor(s,dtype=self.platform.float).to(self.nn.device) # convert s to a tensor with dtype matching platform.float and move it to the device - a=(self.nn.actor(s)+self.nn.noise()).detach().numpy() # get the action using the actor network and add some noise and convert it to numpy array - except Exception as e: - raise e # raise the exception if any error occurs - next_s,r,done=self.nn.env(a) # get the next state, reward, and done flag from the environment using the custom env function and the chosen action - try: - if self.platform.DType!=None: # check if the platform is TensorFlow - if type(self.nn.param[0])!=list: - next_s=np.array(next_s,self.nn.param[0].dtype.name) # convert next_s to an array with dtype matching nn.param[0] - r=np.array(r,self.nn.param[0].dtype.name) # convert r to an array with dtype matching nn.param[0] - done=np.array(done,self.nn.param[0].dtype.name) # convert done to an array with dtype matching nn.param[0] - else: - next_s=np.array(next_s,self.nn.param[0][0].dtype.name) # convert next_s to an array with dtype matching nn.param[0][0] - r=np.array(r,self.nn.param[0][0].dtype.name) # convert r to an array with dtype matching nn.param[0][0] - done=np.array(done,self.nn.param[0][0].dtype.name) # convert done to an array with dtype matching nn.param[0][0] - except Exception: - pass - try: - self.nn.pool(self.state_pool,self.action_pool,self.next_state_pool,self.reward_pool,self.done_pool,[s,a,next_s,reward,done]) # add the data to the pool using the custom pool function - except Exception as e: - try: - if self.nn.pool!=None: # check if the custom pool function is defined or not - raise e # raise the exception if it is defined - except Exception: - self.pool(s,a,next_s,r,done) # add the data to the pool using the default pool function - try: - self.nn.pr.TD=np.append(self.nn.pr.TD,self.nn.initial_TD) # append the initial TD error value to the TD error list for prioritized replay buffer - if len(self.state_pool)>self.pool_size: # check if the pool size exceeds the maximum size - TD=np.array(0) # create a zero TD error value - self.nn.pr.TD=np.append(TD,self.nn.pr.TD[2:]) # remove the oldest TD error value from the TD error list and append the zero TD error value - except Exception as e: - try: - if self.nn.pr!=None: # check if the prioritized replay buffer is defined or not - raise e # raise the exception if it is defined - except Exception: - pass - self.reward=r+self.reward # accumulate the reward value over steps - loss=self._train() # train the network using the pool data and get the loss value - self.sc+=1 # increase the step counter by one - if done: # check if done flag is True or not - if self.save_episode==True: # check if save episode flag is True or not - episode=[s,a,next_s,r] # create a list of data for this step - self.reward_list.append(self.reward) # append the reward value to the reward list - return loss,episode,done # return the loss value, episode data, and done flag - elif self.save_episode==True: # check if save episode flag is True or not - episode=[s,a,next_s,r] # create a list of data for this step - s=next_s # update s as next_s for next step - self.reward_list.append(self.reward) # append the reward value to the reward list - return loss,episode,done # return the loss value, episode data, and done flag - - + # increment bc attribute of nn by 1 + self.nn.bc+=1 + # increment j by 1 + j+=1 + # divide loss by j and assign it back to loss + loss=loss/j + # return loss + return loss + + # define a method named train_ + def train_(self): + # initialize the reward attribute as 0 + self.reward=0 + # initialize the episode variable as None + episode=None + # if the nn attribute has a reset method, execute the following statements + if hasattr(self.nn,'reset'): + # call the reset method of the nn attribute + self.nn.reset() + # if the nn attribute has a env attribute, execute the following statements + if hasattr(self.nn,'env'): + # assign the output of the env attribute's reset method to the s variable + s=self.nn.env.reset() + # else, execute the following statements + else: + # assign the output of the genv attribute's reset method to the s variable + s=self.genv.reset() + # if the episode_step attribute is not None, execute the following statements + if self.episode_step!=None: + # use a for loop to iterate from 0 to episode_step + for _ in range(self.episode_step): + # call the suspend_func method + self.suspend_func() + # assign the output of the epsilon_greedy_policy method with s as input to the action_prob variable + action_prob=self.epsilon_greedy_policy(s) + # assign a random choice from range of action_count with action_prob as probability to the a variable + a=np.random.choice(range(self.action_count),p=action_prob) + # if the nn attribute has a env attribute, execute the following statements + if hasattr(self.nn,'env'): + # assign the output of the env attribute's step method with a as input to the next_s, r, done variables + next_s,r,done=self.nn.env(a) + # else, execute the following statements + else: + # assign the output of the genv attribute's step method with a as input to the next_s, r, done variables + next_s,r,done,_=self.genv.step(a) + # if hasattr(self.platform,'DType'): + if hasattr(self.platform,'DType'): + # if type of first element of param attribute of nn is not list, execute following statements + if type(self.nn.param[0])!=list: + # convert next_s to numpy array with dtype name of first element of param attribute of nn and assign it back to next_s + next_s=np.array(next_s,self.nn.param[0].dtype.name) + # convert r to numpy array with dtype name of first element of param attribute of nn and assign it back to r + r=np.array(r,self.nn.param[0].dtype.name) + # convert done to numpy array with dtype name of first element of param attribute of nn and assign it back to done + done=np.array(done,self.nn.param[0].dtype.name) + # else, execute following statements + else: + # convert next_s to numpy array with dtype name of first element of first element of param attribute of nn and assign it back to next_s + next_s=np.array(next_s,self.nn.param[0][0].dtype.name) + # convert r to numpy array with dtype name of first element of first element of param attribute of nn and assign it back to r + r=np.array(r,self.nn.param[0][0].dtype.name) + # convert done to numpy array with dtype name of first element of first element of param attribute of nn and assign it back to done + done=np.array(done,self.nn.param[0][0].dtype.name) + # if the nn attribute has a pool method, execute the following statements + if hasattr(self.nn,'pool'): + # call the pool method of the nn attribute with the state_pool, action_pool, next_state_pool, reward_pool, done_pool, s, a, next_s, r, done as inputs + self.nn.pool(self.state_pool,self.action_pool,self.next_state_pool,self.reward_pool,self.done_pool,[s,a,next_s,r,done]) + # else, execute the following statements + else: + # call the pool method with s, a, next_s, r, done as inputs + self.pool(s,a,next_s,r,done) + # if the nn attribute has a pr attribute, execute the following statements + if hasattr(self.nn,'pr'): + # append the initial_TD attribute of the nn attribute to the TD attribute of the pr attribute of the nn attribute + self.nn.pr.TD=np.append(self.nn.pr.TD,self.nn.initial_TD) + # if the length of the state_pool attribute is greater than the pool_size attribute, execute the following statements + if len(self.state_pool)>self.pool_size: + # assign a numpy array of 0 to the TD variable + TD=np.array(0) + # append TD to the TD attribute of the pr attribute of the nn attribute from index 2 onwards + self.nn.pr.TD=np.append(TD,self.nn.pr.TD[2:]) + # add r to the reward attribute and assign it back to the reward attribute + self.reward=r+self.reward + # assign the output of the _train method to the loss variable + loss=self._train() + # increment the sc attribute by 1 + self.sc+=1 + # if done is True, execute the following statements + if done: + # if the save_episode attribute is True, execute the following statements + if self.save_episode==True: + # append a list of s, a, next_s, r to the episode variable + episode=[s,a,next_s,r] + # append the reward attribute to the reward_list attribute + self.reward_list.append(self.reward) + # return loss, episode, done + return loss,episode,done + # else if save_episode is True, execute following statements + elif self.save_episode==True: + # append list of s, a, next_s, r to episode variable + episode=[s,a,next_s,r] + # assign next_s to s + s=next_s + # else, execute following statements + else: + # use while loop to iterate indefinitely + while True: + # call suspend_func method + self.suspend_func() + # assign output from epsilon_greedy_policy method with s as input to action_prob variable + action_prob=self.epsilon_greedy_policy(s) + # assign random choice from range of action_count with action_prob as probability to a variable + a=np.random.choice(range(self.action_count),p=action_prob) + # if nn has env attribute, execute following statements + if hasattr(self.nn,'env'): + # assign output from env method from nn with a as input to next_s, r, done variables + next_s,r,done=self.nn.env(a) + # else, execute following statements + else: + # assign output from step method from genv with a as input to next_s, r, done variables + next_s,r,done,_=self.genv.step(a) + # if hasattr(self.platform,'DType'): + if hasattr(self.platform,'DType'): + # if type of first element of param attribute of nn is not list, execute following statements + if type(self.nn.param[0])!=list: + # convert next_s to numpy array with dtype name of first element of param attribute of nn and assign it back to next_s + next_s=np.array(next_s,self.nn.param[0].dtype.name) + # convert r to numpy array with dtype name of first element of param attribute of nn and assign it back to r + r=np.array(r,self.nn.param[0].dtype.name) + # convert done to numpy array with dtype name of first element of param attribute of nn and assign it back to done + done=np.array(done,self.nn.param[0].dtype.name) + # else, execute following statements + else: + # convert next_s to numpy array with dtype name of first element of first element of param attribute of nn and assign it back to next_s + next_s=np.array(next_s,self.nn.param[0][0].dtype.name) + # convert r to numpy array with dtype name of first element of first element of param attribute of nn and assign it back to r + r=np.array(r,self.nn.param[0][0].dtype.name) + # convert done to numpy array with dtype name of first element of first element of param attribute of nn and assign it back to done + done=np.array(done,self.nn.param[0][0].dtype.name) + # if the nn attribute has a pool method, execute the following statements + if hasattr(self.nn,'pool'): + # call the pool method of the nn attribute with the state_pool, action_pool, next_state_pool, reward_pool, done_pool, s, a, next_s, r, done as inputs + self.nn.pool(self.state_pool,self.action_pool,self.next_state_pool,self.reward_pool,self.done_pool,[s,a,next_s,r,done]) + # else, execute the following statements + else: + # call the pool method with s, a, next_s, r, done as inputs + self.pool(s,a,next_s,r,done) + # if the nn attribute has a pr attribute, execute the following statements + if hasattr(self.nn,'pr'): + # append the initial_TD attribute of the nn attribute to the TD attribute of the pr attribute of the nn attribute + self.nn.pr.TD=np.append(self.nn.pr.TD,self.nn.initial_TD) + # if the length of the state_pool attribute is greater than the pool_size attribute, execute the following statements + if len(self.state_pool)>self.pool_size: + # assign a numpy array of 0 to the TD variable + TD=np.array(0) + # append TD to the TD attribute of the pr attribute of the nn attribute from index 2 onwards + self.nn.pr.TD=np.append(TD,self.nn.pr.TD[2:]) + # add r to the reward attribute and assign it back to the reward attribute + self.reward=r+self.reward + # assign the output of the _train method to the loss variable + loss=self._train() + # increment the sc attribute by 1 + self.sc+=1 + # if done is True or save_episode is True, execute the following statements + if done or self.save_episode==True: + # initialize episode variable as an empty list + episode=[] + # use a for loop to iterate over range from 0 to sc plus 1 + for i in range(0,self.sc+1): + # append a list of state_pool[i], action_pool[i], next_state_pool[i], reward_pool[i] to episode variable + episode.append([self.state_pool[i],self.action_pool[i],self.next_state_pool[i],self.reward_pool[i]]) + # if done is True, append 'done' string to episode variable + if done: + episode.append('done') + # return loss, episode, done + return loss,episode,done + # assign next_s to s + s=next_s + + # define a method named train def train(self,episode_count,save=None,one=True,p=None,s=None): - avg_reward=None # initialize the average reward value as None - if p==None: # check if p is defined or not - self.p=9 # set p as 9 by default + # initialize the average reward variable as None + avg_reward=None + # if p is None, assign 9 to the p attribute + if p==None: + self.p=9 + # else, assign p minus 1 to the p attribute else: - self.p=p-1 # decrease p by one - if s==None: # check if s is defined or not - self.s=1 # set s as 1 by default - self.file_list=None # set file_list as None by default + self.p=p-1 + # if s is None, assign 1 to the s attribute and None to the file_list attribute + if s==None: + self.s=1 + self.file_list=None + # else, assign s minus 1 to the s attribute and an empty list to the file_list attribute else: - self.s=s-1 # decrease s by one - self.file_list=[] # initialize an empty list for file_list - if episode_count!=None: # check if episode_count is defined or not - for i in range(episode_count): # loop over each episode - t1=time.time() # record the start time of the episode - loss,episode,done=self.train_() # train the network for one episode and get the loss value, episode data, and done flag - if self.trial_count!=None: # check if trial_count is defined or not - if len(self.reward_list)>=self.trial_count: # check if the reward list has enough values for trial_count - avg_reward=statistics.mean(self.reward_list[-self.trial_count:]) # calculate the average reward value using the last trial_count values in the reward list - if self.criterion!=None and avg_reward>=self.criterion: # check if criterion is defined or not and if the average reward value meets the criterion or not - t2=time.time() # record the end time of the episode - self.total_time+=(t2-t1) # accumulate the total time over episodes - self._time=self.total_time-int(self.total_time) # get the decimal part of the total time - if self._time<0.5: - self.total_time=int(self.total_time) # round down the total time if the decimal part is less than 0.5 + self.s=s-1 + self.file_list=[] + # if episode_count is not None, execute the following statements + if episode_count!=None: + # use a for loop to iterate from 0 to episode_count + for i in range(episode_count): + # record the current time as t1 + t1=time.time() + # call the train_ method and assign the outputs to loss, episode, and done variables + loss,episode,done=self.train_() + # if the trial_count attribute is not None, execute the following statements + if self.trial_count!=None: + # if the length of the reward_list attribute is greater than or equal to the trial_count attribute, execute the following statements + if len(self.reward_list)>=self.trial_count: + # calculate the mean of the last trial_count elements of the reward_list attribute and assign it to avg_reward + avg_reward=statistics.mean(self.reward_list[-self.trial_count:]) + # if the criterion attribute is not None and avg_reward is greater than or equal to the criterion attribute, execute the following statements + if self.criterion!=None and avg_reward>=self.criterion: + # record the current time as t2 + t2=time.time() + # add t2 minus t1 to the total_time attribute and assign it back to itself + self.total_time+=(t2-t1) + # assign total_time minus its integer part to time_ attribute + time_=self.total_time-int(self.total_time) + # if time_ is less than 0.5, assign the integer part of total_time to itself + if time_<0.5: + self.total_time=int(self.total_time) + # else, assign the integer part of total_time plus 1 to itself else: - self.total_time=int(self.total_time)+1 # round up the total time if the decimal part is greater than or equal to 0.5 - print('episode:{0}'.format(self.total_episode)) # print the total number of episodes - print('last loss:{0:.6f}'.format(loss)) # print the last loss value - print('average reward:{0}'.format(avg_reward)) # print the average reward value + self.total_time=int(self.total_time)+1 + # print the total_episode attribute + print('episode:{0}'.format(self.total_episode)) + # print the loss with six decimal places + print('last loss:{0:.6f}'.format(loss)) + # print the avg_reward + print('average reward:{0}'.format(avg_reward)) print() - print('time:{0}s'.format(self.total_time)) # print the total time in seconds - return # stop training and return - self.loss=loss # update loss as the last loss value - self.loss_list.append(loss) # append loss to the loss list - self.total_episode+=1 # increase the total episode counter by one - if episode_count%10!=0: - p=episode_count-episode_count%self.p - p=int(p/self.p) - s=episode_count-episode_count%self.s - s=int(s/self.s) - else: - p=episode_count/(self.p+1) - p=int(p) - s=episode_count/(self.s+1) - s=int(s) - if p==0: - p=1 - if s==0: - s=1 - if i%p==0: # check if i is a multiple of p or not - if len(self.state_pool)>=self.batch: # check if the pool size is larger than or equal to the batch size or not - print('episode:{0} loss:{1:.6f}'.format(i+1,loss)) # print the episode number and loss value - if avg_reward!=None: # check if avg_reward is None or not - print('episode:{0} average reward:{1}'.format(i+1,avg_reward)) # print the episode number and average reward value + # print the total_time with 's' as unit + print('time:{0}s'.format(self.total_time)) + # return nothing + return + # assign the loss to the loss attribute + self.loss=loss + # append the loss to the loss_list attribute + self.loss_list.append(loss) + # increment the total_episode attribute by 1 + self.total_episode+=1 + # if episode_count is not None and episode_count is divisible by 10, execute the following statements + if episode_count!=None and episode_count%10==0: + # assign episode_count divided by p plus 1 to p and convert it to an integer + p=int(episode_count/(self.p+1)) + # assign episode_count divided by s plus 1 to s and convert it to an integer + s=int(episode_count/(self.s+1)) + # else, execute the following statements + else: + # assign episode_count minus its remainder when divided by p to p and divide it by p and convert it to an integer + p=int((episode_count-episode_count%self.p)/self.p) + # assign episode_count minus its remainder when divided by s to s and divide it by s and convert it to an integer + s=int((episode_count-episode_count%s)/self.s) + # if p is 0, assign 1 to p + if p==0: + p=1 + # if s is 0, assign 1 to s + if s==0: + s=1 + # if i plus 1 is divisible by p, execute the following statements + if (i+1)%p==0: + # if the length of the state_pool attribute is greater than or equal to the batch attribute, execute the following statements + if len(self.state_pool)>=self.batch: + # print i plus 1 and loss with six decimal places + print('episode:{0} loss:{1:.6f}'.format(i+1,loss)) + # if avg_reward is not None, execute the following statements + if avg_reward!=None: + # print i plus 1 and avg_reward + print('episode:{0} average reward:{1}'.format(i+1,avg_reward)) + # else, execute the following statements else: - print('episode:{0} reward:{1}'.format(i+1,self.reward)) # print the episode number and reward value + # print i plus 1 and reward + print('episode:{0} reward:{1}'.format(i+1,self.reward)) print() - if save!=None and i%s==0: # check if save is None or not and if i is a multiple of s orif save!=None and i%s==0: # check if save is None or not and if i is a multiple of s or not - self.save(self.total_episode,one) # save the network parameters and training data using the custom save function and the total episode number and one flag - if self.save_episode==True: # check if save episode flag is True or not - if done: # check if done flag is True or not - episode.append('done') # append 'done' to the episode data list - self.episode_set.append(episode) # append the episode data list to the episode set list - if self.max_episode_count!=None and len(self.episode_set)>=self.max_episode_count: # check if max episode count is defined or not and if the episode set size reaches the max episode count or not - self.save_episode=False # set the save episode flag as False to stop saving episodes + # if save is not None and i plus 1 is divisible by s, execute the following statements + if save!=None and (i+1)%s==0: + # call the save method with total_episode and one as inputs + self.save(self.total_episode,one) + # if save_episode is True, execute the following statements + if self.save_episode==True: + # append episode to the episode_set attribute + self.episode_set.append(episode) + # if max_episode_count is not None and length of episode_set attribute is greater than or equal to max_episode_count attribute, execute the following statements + if self.max_episode_count!=None and len(self.episode_set)>=self.max_episode_count: + # assign False to save_episode attribute + self.save_episode=False try: try: - self.nn.ec.assign_add(1) # increase the episode counter by one using TensorFlow assign_add method + # call the assign_add method of the ec attribute of the nn attribute with 1 as input + self.nn.ec.assign_add(1) except Exception: - self.nn.ec+=1 # increase the episode counter by one using Python addition operator + # increment the ec attribute of the nn attribute by 1 + self.nn.ec+=1 except Exception: - pass - t2=time.time() # record the end time of the episode - self.time+=(t2-t1) # accumulate the time over episodes - else: - i=0 # initialize the episode index as zero - while True: # loop until an exception occurs or the criterion is met - t1=time.time() # record the start time of the episode - loss,episode,done=self.train_() # train the network for one episode and get the loss value, episode data, and done flag - if self.trial_count!=None: # check if trial_count is defined or not - if len(self.reward_list)==self.trial_count: # check if the reward list has enough values for trial_count - avg_reward=statistics.mean(self.reward_list[-self.trial_count:]) # calculate the average reward value using the last trial_count values in the reward list - if avg_reward>=self.criterion: # check if the average reward value meets the criterion or not - t2=time.time() # record the end time of the episode - self.total_time+=(t2-t1) # accumulate the total time over episodes - self._time=self.total_time-int(self.total_time) # get the decimal part of the total time - if self._time<0.5: - self.total_time=int(self.total_time) # round down the total time if the decimal part is less than 0.5 + pass + t2=time.time() + self.time+=(t2-t1) + # else, execute the following statements + else: + # initialize i as 0 + i=0 + # use a while loop to iterate indefinitely + while True: + # record the current time as t1 + t1=time.time() + # call the train_ method and assign the outputs to loss, episode, and done variables + loss,episode,done=self.train_() + # if the trial_count attribute is not None, execute the following statements + if self.trial_count!=None: + # if the length of the reward_list attribute is equal to the trial_count attribute, execute the following statements + if len(self.reward_list)==self.trial_count: + # calculate the mean of the last trial_count elements of the reward_list attribute and assign it to avg_reward + avg_reward=statistics.mean(self.reward_list[-self.trial_count:]) + # if avg_reward is greater than or equal to the criterion attribute, execute the following statements + if avg_reward>=self.criterion: + # record the current time as t2 + t2=time.time() + # add t2 minus t1 to the total_time attribute and assign it back to itself + self.total_time+=(t2-t1) + # assign total_time minus its integer part to time_ attribute + time_=self.total_time-int(self.total_time) + # if time_ is less than 0.5, assign the integer part of total_time to itself + if time_<0.5: + self.total_time=int(self.total_time) + # else, assign the integer part of total_time plus 1 to itself else: - self.total_time=int(self.total_time)+1 # round up the total time if the decimal part is greater than or equal to 0.5 - print('episode:{0}'.format(self.total_episode)) # print the total number of episodes - print('last loss:{0:.6f}'.format(loss)) # print the last loss value - print('average reward:{0}'.format(avg_reward)) # print the average reward value + self.total_time=int(self.total_time)+1 + # print the total_episode attribute + print('episode:{0}'.format(self.total_episode)) + # print the loss with six decimal places + print('last loss:{0:.6f}'.format(loss)) + # print the avg_reward + print('average reward:{0}'.format(avg_reward)) print() - print('time:{0}s'.format(self.total_time)) # print the total time in seconds - return # stop training and return - self.loss=loss # update loss as the last loss value - self.loss_list.append(loss) # append loss to the loss list - i+=1 # increase the episode index by one - self.total_episode+=1 # increase the total episode counter by one - if episode_count%10!=0: - p=episode_count-episode_count%self.p - p=int(p/self.p) - s=episode_count-episode_count%self.s - s=int(s/self.s) - else: - p=episode_count/(self.p+1) - p=int(p) - s=episode_count/(self.s+1) - s=int(s) - if p==0: - p=1 - if s==0: - s=1 - if i%p==0: # check if i is a multiple of p or not - if len(self.state_pool)>=self.batch: # check if the pool size is larger than or equal to the batch size or not - print('episode:{0} loss:{1:.6f}'.format(i+1,loss)) # print the episode number and loss value - if avg_reward!=None: # check if avg_reward is None or not - print('episode:{0} average reward:{1}'.format(i+1,avg_reward)) # print the episode number and average reward valueelse: - else: - print('episode:{0} reward:{1}'.format(i+1,self.reward)) # print the episode number and reward value + # print the total_time with 's' as unit + print('time:{0}s'.format(self.total_time)) + # return nothing + return + # assign the loss to the loss attribute + self.loss=loss + # append the loss to the loss_list attribute + self.loss_list.append(loss) + # increment i by 1 + i+=1 + # increment the total_episode attribute by 1 + self.total_episode+=1 + # if episode_count is not None and episode_count is not divisible by 10, execute the following statements + if episode_count!=None and episode_count%10!=0: + # assign episode_count minus its remainder when divided by p to p and divide it by p and convert it to an integer + p=int((episode_count-episode_count%self.p)/self.p) + # assign episode_count minus its remainder when divided by s to s and divide it by s and convert it to an integer + s=int((episode_count-episode_count%s)/self.s) + # else, execute the following statements + else: + # assign episode_count divided by p plus 1 to p and convert it to an integer + p=int(episode_count/(self.p+1)) + # assign episode_count divided by s plus 1 to s and convert it to an integer + s=int(episode_count/(self.s+1)) + # if p is 0, assign 1 to p + if p==0: + p=1 + # if s is 0, assign 1 to s + if s==0: + s=1 + # if i plus 1 is divisible by p, execute the following statements + if (i+1)%p==0: + # if the length of the state_pool attribute is greater than or equal to the batch attribute, execute the following statements + if len(self.state_pool)>=self.batch: + # print i plus 1 and loss with six decimal places + print('episode:{0} loss:{1:.6f}'.format(i+1,loss)) + # if avg_reward is not None, execute the following statements + if avg_reward!=None: + # print i plus 1 and avg_reward + print('episode:{0} average reward:{1}'.format(i+1,avg_reward)) + # else, execute the following statements + else: + # print i plus 1 and reward + print('episode:{0} reward:{1}'.format(i+1,self.reward)) print() - if save!=None and i%s==0: # check if save is None or not and if i is a multiple of s or not - self.save(self.total_episode,one) # save the network parameters and training data using the custom save function and the total episode number and one flag - if self.save_episode==True: # check if save episode flag is True or not - if done: # check if done flag is True or not - episode.append('done') # append 'done' to the episode data list - self.episode_set.append(episode) # append the episode data list to the episode set list - if self.max_episode_count!=None and len(self.episode_set)>=self.max_episode_count: # check if max episode count is defined or not and if the episode set size reaches the max episode count or not - self.save_episode=False # set the save episode flag as False to stop saving episodes - try: + # if save is not None and i plus 1 is divisible by s, execute the following statements + if save!=None and (i+1)%s==0: + # call the save method with total_episode and one as inputs + self.save(self.total_episode,one) + # if save_episode is True, execute the following statements + if self.save_episode==True: + # if done is True, append 'done' string to episode + if done: + episode.append('done') + # append episode to the episode_set attribute + self.episode_set.append(episode) + # if max_episode_count is not None and length of episode_set attribute is greater than or equal to max_episode_count attribute, execute the following statements + if self.max_episode_count!=None and len(self.episode_set)>=self.max_episode_count: + # assign False to save_episode attribute + self.save_episode=False + # if the nn attribute has an ec attribute, execute the following statements + if hasattr(self.nn,'ec'): + # try to execute the following statements try: - self.nn.ec.assign_add(1) # increase the episode counter by one using TensorFlow assign_add method + # call the assign_add method of the ec attribute of the nn attribute with 1 as input + self.nn.ec.assign_add(1) + # if an exception occurs, execute the following statements except Exception: - self.nn.ec+=1 # increase the episode counter by one using Python addition operator - except Exception: - pass - t2=time.time() # record the end time of the episode - self.time+=(t2-t1) # accumulate the time over episodes - self._time=self.time-int(self.time) # get the decimal part of the time - if self._time<0.5: - self.total_time=int(self.time) # round down the time if the decimal part is less than 0.5 - else: - self.total_time=int(self.time)+1 # round up the time if the decimal part is greater than or equal to 0.5 - self.total_time+=self.time # add the time to the total time - print('last loss:{0:.6f}'.format(loss)) # print the last loss value - print('last reward:{0}'.format(self.reward)) # print the last reward value - print() - print('time:{0}s'.format(self.time)) # print the time in seconds - return # return - - - def visualize_reward(self): - print() - plt.figure(1) # create a figure object with number 1 - plt.plot(np.arange(self.total_episode),self.reward_list) # plot the reward list against the episode number - plt.xlabel('episode') # set the x-axis label as 'episode' - plt.ylabel('reward') # set the y-axis label as 'reward' - print('reward:{0:.6f}'.format(self.reward_list[-1])) # print the last reward value - return - - - def visualize_train(self): - print() - plt.figure(1) # create a figure object with number 1 - plt.plot(np.arange(self.total_episode),self.loss_list) # plot the loss list against the episode number - plt.title('train loss') # set the title of the figure as 'train loss' - plt.xlabel('episode') # set the x-axis label as 'episode' - plt.ylabel('loss') # set the y-axis label as 'loss' - print('loss:{0:.6f}'.format(self.loss_list[-1])) # print the last loss value - return + # increment the ec attribute of the nn attribute by 1 + self.nn.ec+=1 + # record the current time as t2 + t2=time.time() + # add t2 minus t1 to the time attribute and assign it back to itself + self.time+=(t2-t1) + # assign time minus its integer part to time_ attribute + time_=self.time-int(self.time) + # if time_ is less than 0.5, assign the integer part of time to itself + if time_<0.5: + self.total_time=int(self.time) + # else, assign the integer part of time plus 1 to itself + else: + self.total_time=int(self.time)+1 + # add time to the total_time attribute and assign it back to itself + self.total_time+=self.time + # print the loss with six decimal places + print('last loss:{0:.6f}'.format(loss)) + # print the reward + print('last reward:{0}'.format(self.reward)) + print() + # print the time with 's' as unit + print('time:{0}s'.format(self.time)) + # return nothing + return + + # define a method named train_online + def train_online(self): + # use while loop to iterate indefinitely + while True: + # if hasattr(nn,'save'): + if hasattr(self.nn,'save'): + # call save method of nn with save as input + self.nn.save(self.save) + # if hasattr(nn,'stop_flag'): + if hasattr(self.nn,'stop_flag'): + # if stop_flag attribute of nn is True, return nothing + if self.nn.stop_flag==True: + return + # if hasattr(nn,'stop_func'): + if hasattr(self.nn,'stop_func'): + # if output of stop_func method of nn is True, return nothing + if self.nn.stop_func(): + return + # if hasattr(nn,'suspend_func'): + if hasattr(self.nn,'suspend_func'): + # call suspend_func method of nn + self.nn.suspend_func() + # assign output from online method of nn to data variable + data=self.nn.online() + # if data is 'stop', return nothing + if data=='stop': + return + # elif data is 'suspend', execute following statements + elif data=='suspend': + # call suspend_func method of nn + self.nn.suspend_func() + # assign output from opt_ol method with first, second, third, fourth, fifth elements of data as inputs to loss variable + loss=self.opt_ol(data[0],data[1],data[2],data[3],data[4]) + # convert loss to numpy array and assign it back to loss + loss=loss.numpy() + # append loss to train_loss_list attribute of nn + self.nn.train_loss_list.append(loss) + # if length of train_acc_list attribute of nn equals max_length attribute of nn, delete first element of train_acc_list attribute of nn + if len(self.nn.train_acc_list)==self.nn.max_length: + del self.nn.train_acc_list[0] + # if hasattr(nn,'counter'): + if hasattr(self.nn,'counter'): + # increment counter attribute of nn by 1 + self.nn.counter+=1 + # return nothing + return From e09f1543f94fd88340b32612af7d35dd4f660266 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Sat, 22 Jul 2023 22:31:59 +0800 Subject: [PATCH 073/337] Update kernel.py --- Note 7.0 documentation/RL/kernel/kernel.py | 1209 ++++++++++---------- 1 file changed, 606 insertions(+), 603 deletions(-) diff --git a/Note 7.0 documentation/RL/kernel/kernel.py b/Note 7.0 documentation/RL/kernel/kernel.py index abbdc720..65ef15d1 100644 --- a/Note 7.0 documentation/RL/kernel/kernel.py +++ b/Note 7.0 documentation/RL/kernel/kernel.py @@ -1,669 +1,672 @@ +# This is a code snippet that defines a kernel class for multi-process reinforcement learning +# using TensorFlow and multiprocessing + import tensorflow as tf from tensorflow.python.ops import state_ops from tensorflow.python.util import nest from multiprocessing import Value,Array import numpy as np -import matplotlib.pyplot as plt import statistics class kernel: def __init__(self,nn=None,process=None): self.nn=nn # the neural network model - if process!=None: # the number of processes + if process!=None: self.reward=np.zeros(process,dtype=np.float32) # the reward array for each process self.sc=np.zeros(process,dtype=np.int32) # the step counter array for each process - self.state_pool={} # the dictionary to store the state pool for each process - self.action_pool={} # the dictionary to store the action pool for each process - self.next_state_pool={} # the dictionary to store the next state pool for each process - self.reward_pool={} # the dictionary to store the reward pool for each process - self.done_pool={} # the dictionary to store the done flag pool for each process - self.epsilon=None # the epsilon value for epsilon-greedy policy + self.state_pool={} # the state pool dictionary for each process + self.action_pool={} # the action pool dictionary for each process + self.next_state_pool={} # the next state pool dictionary for each process + self.reward_pool={} # the reward pool dictionary for each process + self.done_pool={} # the done pool dictionary for each process + self.epsilon=None # the exploration rate array for each process self.episode_step=None # the maximum number of steps per episode - self.pool_size=None # the maximum size of the experience pool + self.pool_size=None # the maximum size of the pool self.batch=None # the batch size for training - self.episode=0 # the episode counter - self.update_step=None # the frequency of updating the target network parameters - self.trial_count=None # the number of trials to calculate the average reward + self.episode_=0 # the episode counter + self.update_step=None # the update frequency for the target network + self.trial_count=None # the number of trials to compute the average reward self.process=process # the number of processes - self.process_counter=0 # the counter of running processes - self.probability_list=[] # the list to store the probability distribution of running processes - self.running_flag_list=[] # the list to store the running flag of each process - self.finish_list=[] # the list to store the finished processes - self.running_flag=[] # the list to store the running flag of all processes (including 0) - self.PO=None # the optimization strategy (1: lock, 2: global lock, 3: no lock) - self.priority_flag=False # whether to use priority optimization or not - self.priority_p=0 # the priority process index for optimization - self.max_opt=None # the maximum number of optimization steps per process before switching priority - self.stop=False # whether to stop training or not - self.save_flag=False # whether to save the model or not - self.stop_flag=False # whether to stop all processes or not - self.reward_list=[] # the list to store the total reward per episode - self.loss_list=[] # the list to store the average loss per episode - self.total_episode=0 # the total number of episodes + self.process_counter=0 # the running process counter + self.probability_list=[] # the probability list for sampling processes based on their running flag + self.running_flag_list=[] # the running flag list for each process + self.finish_list=[] # the finish flag list for each process + self.running_flag=[] # the running flag array for each process + self.PO=None # the parallel optimization mode (1, 2, or 3) + self.priority_flag=False # the priority flag for optimization order + self.priority_p=0 # the priority index for optimization order + self.max_opt=None # the maximum number of optimization steps per process per episode + self.stop=False # the stop flag for training + self.save_flag=False # the save flag for saving parameters to file + self.stop_flag=False # the stop flag for optimization operation + self.opt_counter=None # the optimization counter array for each process + self.s=None # the state array for online training mode + self.filename='save.dat' # the file name to save parameters to file + self.reward_list=[] # the reward list to store episode rewards + self.loss_list=[] # the loss list to store episode losses + self.total_episode=0 # the total episode counter def init(self,manager): - self.state_pool=manager.dict(self.state_pool) # use manager.dict to share state pool among processes - self.action_pool=manager.dict(self.action_pool) # use manager.dict to share action pool among processes - self.next_state_pool=manager.dict(self.next_state_pool) # use manager.dict to share next state pool among processes - self.reward_pool=manager.dict(self.reward_pool) # use manager.dict to share reward pool among processes - self.done_pool=manager.dict(self.done_pool) # use manager.dict to share done flag pool among processes - self.reward=Array('f',self.reward) # use Array to share reward array among processes - if type(self.nn.param[0])!=list: # the loss array for each process - self.loss=np.zeros(self.process,dtype=self.nn.param[0].dtype.name) + self.state_pool=manager.dict(self.state_pool) # create a shared memory space for state pool using manager object + self.action_pool=manager.dict(self.action_pool) # create a shared memory space for action pool using manager object + self.next_state_pool=manager.dict(self.next_state_pool) # create a shared memory space for next state pool using manager object + self.reward_pool=manager.dict(self.reward_pool) # create a shared memory space for reward pool using manager object + self.done_pool=manager.dict(self.done_pool) # create a shared memory space for done pool using manager object + self.reward=Array('f',self.reward) # create a shared memory space for reward array using Array object + if type(self.nn.param[0])!=list: + self.loss=np.zeros(self.process,dtype=self.nn.param[0].dtype.name) # create a loss array with the same data type as neural network parameters else: - self.loss=np.zeros(self.process,dtype=self.nn.param[0][0].dtype.name) - self.loss=Array('f',self.loss) # use Array to share loss array among processes - self.sc=Array('f',self.sc) # use Array to share step counter array among processes - self.process_counter=Value('i',self.process_counter) # use Value to share process counter among processes - self.probability_list=manager.list(self.probability_list) # use manager.list to share probability list among processes - self.running_flag_list=manager.list(self.running_flag_list) # use manager.list to share running flag list among processes - self.finish_list=manager.list(self.finish_list) # use manager.list to share finish list among processes - self.running_flag=manager.list([0]) # use manager.list to share running flag of all processes (including 0) - self.reward_list=manager.list(self.reward_list) # use manager.list to store reward list among processes - self.loss_list=manager.list(self.loss_list) # use manager.list to store loss list among processes - self.total_episode=Value('i',self.total_episode) # use Value to share total episode among processes - self.priority_p=Value('i',self.priority_p) # use Value to share priority process index among processes - if self.priority_flag==True: # if priority optimization is enabled - self.opt_counter=Array('i',np.zeros(self.process,dtype=np.int32)) # use Array to store optimization counter for each process + self.loss=np.zeros(self.process,dtype=self.nn.param[0][0].dtype.name) # create a loss array with the same data type as neural network parameters + self.loss=Array('f',self.loss) + self.sc=Array('i',self.sc) # create a shared memory space for step counter array using Array object + self.process_counter=Value('i',self.process_counter) # create a shared memory space for process counter using Value object + self.probability_list=manager.list(self.probability_list) # create a shared memory space for probability list using manager object + self.running_flag_list=manager.list(self.running_flag_list) # create a shared memory space for running flag list using manager object + self.finish_list=manager.list(self.finish_list) # create a shared memory space for finish flag list using manager object + self.running_flag=manager.list([0]) # create a shared memory space for running flag array using manager object + self.reward_list=manager.list(self.reward_list) # create a shared memory space for reward list using manager object + self.loss_list=manager.list(self.loss_list) # create a shared memory space for loss list using manager object + self.total_episode=Value('i',self.total_episode) # create a shared memory space for total episode counter using Value object + self.priority_p=Value('i',self.priority_p) # create a shared memory space for priority index using Value object + if self.priority_flag==True: + self.opt_counter=Array('i',np.zeros(self.process,dtype=np.int32)) # create a shared memory space for optimization counter array using Array object try: - self.nn.opt_counter=manager.list([self.nn.opt_counter]) # use manager.list to share optimization counter for attenuation function among processes + self.nn.opt_counter=manager.list([self.nn.opt_counter]) # create a shared memory space for neural network optimization counter using manager object except Exception: - self.opt_counter_=manager.list() + self.opt_counter_=manager.list() # create an empty list to store the exception try: - self.nn.ec=manager.list(self.nn.ec) # use manager.list to share episode counter for neural network model among processes + self.nn.ec=manager.list([self.nn.ec]) # create a shared memory space for neural network episode counter using manager object except Exception: - self.ec_=manager.list() + self.ec_=manager.list() # create an empty list to store the exception try: - self.nn.bc=manager.list(self.nn.bc) # use manager.list to share batch counter for neural network model among processes + self.nn.bc=manager.list([self.nn.bc]) # create a shared memory space for neural network batch counter using manager object except Exception: - self.bc_=manager.list() - self.stop_flag=Value('b',self.stop_flag) # use Value to share stop flag among processes - self.save_flag=Value('b',self.save_flag) # use Value to share save flag among processes - self.param=manager.dict() # use manager.dict to share parameters among processes + self.bc_=manager.list() # create an empty list to store the exception + self.episode_=Value('i',self.total_episode) # create a shared memory space for episode counter using Value object + self.stop_flag=Value('b',self.stop_flag) # create a shared memory space for stop flag using Value object + self.save_flag=Value('b',self.save_flag) # create a shared memory space for save flag using Value object + self.file_list=manager.list([]) # create an empty list to store the file names + self.param=manager.dict() # create an empty dictionary to store the parameters return - def action_vec(self): # a method to create a vector of ones with the same length as the action space - self.action_one=np.ones(self.action_count,dtype=np.int8) # create a vector of ones with int8 type + def init_online(self,manager): + self.nn.train_loss_list=manager.list([]) # create an empty list to store the training losses + self.nn.counter=manager.list([]) # create an empty list to store the training counters + self.nn.exception_list=manager.list([]) # create an empty list to store the exceptions + self.param=manager.dict() # create an empty dictionary to store the parameters + self.param[7]=self.nn.param # assign the neural network parameters to the dictionary return - def set_up(self,epsilon=None,episode_step=None,pool_size=None,batch=None,update_step=None,trial_count=None,criterion=None): # a method to set up some parameters for the kernel class - if epsilon!=None: # if epsilon value is given - self.epsilon=np.ones(self.process)*epsilon # create an array of epsilon values with the same length as the number of processes - if episode_step!=None: # if episode step value is given - self.episode_step=episode_step # assign the episode step value to the attribute - if pool_size!=None: # if pool size value is given - self.pool_size=pool_size # assign the pool size value to the attribute - if batch!=None: # if batch size value is given - self.batch=batch # assign the batch size value to the attribute - if update_step!=None: # if update step value is given - self.update_step=update_step # assign the update step value to the attribute - if trial_count!=None: # if trial count value is given - self.trial_count=trial_count # assign the trial count value to the attribute - if criterion!=None: # if criterion value is given - self.criterion=criterion # assign the criterion value to the attribute - if epsilon!=None: # if epsilon value is given - self.action_vec() # call the action_vec method to create a vector of ones with the same length as the action space + def action_vec(self): + self.action_one=np.ones(self.action_count,dtype=np.int8) # create an action vector with ones return - def epsilon_greedy_policy(self,s,epsilon): # a method to implement epsilon-greedy policy for action selection - action_prob=self.action_one*epsilon/len(self.action_one) # initialize the action probability vector with uniform distribution multiplied by epsilon - best_a=np.argmax(self.nn.nn.fp(s)) # get the best action index by using the neural network model to predict the Q values for the given state and taking the argmax - action_prob[best_a]+=1-epsilon # increase the probability of the best action by (1-epsilon) - return action_prob - - - def pool(self,s,a,next_s,r,done,pool_lock,index): # a method to store the experience data into the pool for a given process index - pool_lock[index].acquire() # acquire the lock for the process index to avoid race condition - try: - if type(self.state_pool[index])!=np.ndarray and self.state_pool[index]==None: # if the state pool is empty for the process index - self.state_pool[index]=s # assign the state to the state pool - if type(a)==int: # if the action is an integer - if type(self.nn.param[0])!=list: # if the neural network parameter is not a list - a=np.array(a,self.nn.param[0].dtype.name) # convert the action to a numpy array with the same data type as the neural network parameter - else: # if the neural network parameter is a list - a=np.array(a,self.nn.param[0][0].dtype.name) # convert the action to a numpy array with the same data type as the first element of the neural network parameter - self.action_pool[index]=np.expand_dims(a,axis=0) # expand the dimension of the action array and assign it to the action pool - else: # if the action is not an integer - if type(self.nn.param[0])!=list: # if the neural network parameter is not a list - a=a.astype(self.nn.param[0].dtype.name) # convert the action to the same data type as the neural network parameter - else: # if the neural network parameter is a list - a=a.astype(self.nn.param[0][0].dtype.name) # convert the action to the same data type as the first element of the neural network parameter - self.action_pool[index]=a # assign the action to the action pool - self.next_state_pool[index]=np.expand_dims(next_s,axis=0) # expand the dimension of the next state array and assign it to the next state pool - self.reward_pool[index]=np.expand_dims(r,axis=0) # expand the dimension of the reward array and assign it to the reward pool - self.done_pool[index]=np.expand_dims(done,axis=0) # expand the dimension of the done flag array and assign it to the done flag pool - else: # if the state pool is not empty for the process index - try: - self.state_pool[index]=np.concatenate((self.state_pool[index],s),0) # concatenate the state with the existing state pool along axis 0 - if type(a)==int: # if the action is an integer - if type(self.nn.param[0])!=list: # if the neural network parameter is not a list - a=np.array(a,self.nn.param[0].dtype.name) # convert the action to a numpy array with the same data type as the neural network parameter - else: # if the neural network parameter is a list - a=np.array(a,self.nn.param[0][0].dtype.name) # convert the action to a numpy array with the same data type as the first element of the neural network parameter - self.action_pool[index]=np.concatenate((self.action_pool[index],np.expand_dims(a,axis=0)),0) # concatenate the expanded action array with the existing action pool along axis 0 - else: # if the action is not an integer - if type(self.nn.param[0])!=list: # if the neural network parameter is not a list - a=a.astype(self.nn.param[0].dtype.name) # convert the action to the same data type as the neural network parameter - else: # if the neural network parameter is a list - a=a.astype(self.nn.param[0][0].dtype.name) # convert the action to the same data type as the first element of the neural network parameter - self.action_pool[index]=np.concatenate((self.action_pool[index],a),0) # concatenate the action witn the existing action pool along axis 0 - self.next_state_pool[index]=np.concatenate((self.next_state_pool[index],np.expand_dims(next_s,axis=0)),0) # concatenate the expanded next state array witn the existing next state pool along axis 0 - self.reward_pool[index]=np.concatenate((self.reward_pool[index],np.expand_dims(r,axis=0)),0) # concatenate the expanded reward array with the existing reward pool along axis 0 - self.done_pool[index]=np.concatenate((self.done_pool[index],np.expand_dims(done,axis=0)),0) # concatenate the expanded done flag array with the existing done flag pool along axis 0 - except Exception: # if any exception occurs - pass # ignore it and pass - if type(self.state_pool[index])==np.ndarray and len(self.state_pool[index])>self.pool_size: # if the state pool is a numpy array and its length exceeds the pool size - self.state_pool[index]=self.state_pool[index][1:] # remove the oldest state from the state pool - self.action_pool[index]=self.action_pool[index][1:] # remove the oldest action from the action pool - self.next_state_pool[index]=self.next_state_pool[index][1:] # remove the oldest next state from the next state pool - self.reward_pool[index]=self.reward_pool[index][1:] # remove the oldest reward from the reward pool - self.done_pool[index]=self.done_pool[index][1:] # remove the oldest done flag from the done flag pool - except Exception: # if any exception occurs - pool_lock[index].release() # release the lock for the process index - return # return from the method - pool_lock[index].release() # release the lock for the process index - return # return from the method + def set_up(self,epsilon=None,episode_step=None,pool_size=None,batch=None,update_step=None,trial_count=None,criterion=None): + if epsilon!=None: + self.epsilon=np.ones(self.process)*epsilon # assign the exploration rate array with epsilon + if episode_step!=None: + self.episode_step=episode_step # assign the maximum number of steps per episode + if pool_size!=None: + self.pool_size=pool_size # assign the maximum size of the pool + if batch!=None: + self.batch=batch # assign the batch size for training + if update_step!=None: + self.update_step=update_step # assign the update frequency for the target network + if trial_count!=None: + self.trial_count=trial_count # assign the number of trials to compute the average reward + if criterion!=None: + self.criterion=criterion # assign the criterion to judge if the agent solves the environment + if epsilon!=None: + self.action_vec() # create an action vector with ones + return - def get_index(self,p,lock): # a method to get a random process index according to the probability distribution of running processes - while len(self.running_flag_list)self.process_counter.value: # if the lengtn of the running flag list for the current process index is less tnan the process counter value or the sum of the running flag list for the current process index is greater tnan the process counter value - self.running_flag_list[p]=self.running_flag[1:].copy() # assign a copy of the running flag of all processes (excluding 0) to the running flag list for the current process index - while len(self.probability_list)self.pool_size: # check if the pool size exceeds the maximum size + self.state_pool[index]=self.state_pool[index][1:] # delete the oldest state from the pool + self.action_pool[index]=self.action_pool[index][1:] # delete the oldest action from the pool + self.next_state_pool[index]=self.next_state_pool[index][1:] # delete the oldest next state from the pool + self.reward_pool[index]=self.reward_pool[index][1:] # delete the oldest reward from the pool + self.done_pool[index]=self.done_pool[index][1:] # delete the oldest done flag from the pool + except Exception: + pool_lock[index].release() # release the lock for the pool of the current process + return + pool_lock[index].release() # release the lock for the pool of the current process + return + + + def get_index(self,p,lock): + while len(self.running_flag_list)self.process_counter.value: # check if running flag list needs to be updated + self.running_flag_list[p]=self.running_flag[1:].copy() # update running flag list with a copy of running flag array + while len(self.probability_list)=self.trial_count: # if the length of the reward list is greater than or equal to the trial count value - avg_reward=statistics.mean(self.reward_list[-self.trial_count:]) # calculate the average reward of the last trial count episodes - if self.criterion!=None and avg_reward>=self.criterion: # if criterion value is given and average reward is greater than or equal to criterion value - return True # return True to indicate termination condition is met - return False # return False to indicate termination condition is not met + def end(self): + if self.trial_count!=None: # check if trial count is set + if len(self.reward_list)>=self.trial_count: # check if enough rewards are available + avg_reward=statistics.mean(self.reward_list[-self.trial_count:]) # compute the average reward of the last trial count episodes + if self.criterion!=None and avg_reward>=self.criterion: # check if criterion is set and average reward is greater than or equal to criterion + return True # return True to indicate that the agent solves the environment + return False # return False to indicate that the agent does not solve the environment - @tf.function - def opt(self,state_batch,action_batch,next_state_batch,reward_batch,done_batch,p,lock,g_lock=None): # a method to optimize the neural network model for a given batch of experience data, process index and locks - with tf.GradientTape(persistent=True) as tape: # create a persistent gradient tape to record operations on tensors + @tf.function(jit_compile=True) + def opt(self,state_batch,action_batch,next_state_batch,reward_batch,done_batch,p,lock,g_lock=None): + with tf.GradientTape(persistent=True) as tape: # create a gradient tape object to record gradients try: try: - loss=self.nn.loss(state_batch,action_batch,next_state_batch,reward_batch,done_batch) # calculate the loss by using the loss method of the neural network model - except Exception: # if any exception occurs - loss=self.nn.loss(state_batch,action_batch,next_state_batch,reward_batch,done_batch,p) # calculate the loss by using the loss method of the neural network model witn the process index - except Exception as e: # if any exception occurs - raise e # raise the exception again - if self.PO==1: # if the optimization strategy is lock - if self.priority_flag==True and self.priority_p.value!=-1: # if priority optimization is enabled and priority process index is not -1 - while True: # loop until the current process index is equal to the priority process index - if p==self.priority_p.value: # if the current process index is equal to the priority process index - break # break the loop and continue optimization - else: # if the current process index is not equal to the priority process index - continue # skip optimization and continue looping - lock[0].acquire() # acquire the lock for optimization - if self.stop_func_(lock[0]): # call the stop_func_ method witn the lock for optimization and check if it returns True - return 0 # return from the method witn zero value + loss=self.nn.loss(state_batch,action_batch,next_state_batch,reward_batch,done_batch) # compute the loss using neural network model + except Exception: + loss=self.nn.loss(state_batch,action_batch,next_state_batch,reward_batch,done_batch,p) # compute the loss using neural network model and process index + except Exception as e: + raise e # raise any exception + if self.PO==1: # check if parallel optimization mode is 1 + if self.priority_flag==True and self.priority_p.value!=-1: # check if priority flag is True and priority index is not -1 + while True: + if p==self.priority_p.value: # check if current process index is equal to priority index + break # break the loop + else: + continue # try again + lock[0].acquire() # acquire the lock for optimization operation + if self.stop_func_(lock[0]): # check if stop flag is True + return 0 # return 0 to indicate that optimization operation is stopped + if hasattr(self.nn,'gradient'): # check if neural network model has a gradient attribute (custom gradient function) + gradient=self.nn.gradient(tape,loss) # compute the gradient using neural network model and loss + else: + if hasattr(self.nn,'nn'): # check if neural network model has a nn attribute (single network) + gradient=tape.gradient(loss,self.nn.param) # compute the gradient using loss and neural network parameters + else: # otherwise, assume neural network model does not have a nn attribute (actor-critic network) + actor_gradient=tape.gradient(loss[0],self.nn.param[0]) # compute the actor gradient using loss and actor parameters + critic_gradient=tape.gradient(loss[1],self.nn.param[1]) # compute the critic gradient using loss and critic parameters try: - try: - gradient=self.nn.gradient(tape,loss) # calculate the gradient by using the gradient method of the neural network model - except Exception: # if any exception occurs + if hasattr(self.nn,'attenuate'): # check if neural network model has an attenuate attribute (custom attenuate function) try: - if self.nn.nn!=None: # if there is a neural network attribute in the neural network model - gradient=tape.gradient(loss,self.nn.param) # calculate the gradient by using the gradient method of the gradient tape with the loss and the neural network parameter - except Exception: # if any exception occurs - actor_gradient=tape.gradient(loss[0],self.nn.param[0]) # calculate the actor gradient by using the gradient method of the gradient tape with the first element of the loss and the first element of the neural network parameter - critic_gradient=tape.gradient(loss[1],self.nn.param[1]) # calculate the critic gradient by using the gradient method of the gradient tape with the second element of the loss and the second element of the neural network parameter - except Exception as e: # if any exception occurs - raise e # raise the exception again - try: - try: - gradient=self.nn.attenuate(gradient,self.nn.opt_counter,p) # attenuate the gradient by using the attenuate method of the neural network model with the gradient, the optimization counter and the process index - except Exception: # if any exception occurs - actor_gradient=self.nn.attenuate(actor_gradient,self.nn.opt_counter,p) # attenuate the actor gradient by using the attenuate method of the neural network model with the actor gradient, the optimization counter and the process index - critic_gradient=self.nn.attenuate(critic_gradient,self.nn.opt_counter,p) # attenuate the critic gradient by using the attenuate method of the neural network model with the critic gradient, the optimization counter and the process index - except Exception as e: # if any exception occurs - try: - if self.nn.attenuate!=None: # if attenuation function is defined - raise e # raise the exception again - except Exception: # if attenuation function is not defined - pass # ignore it and pass + gradient=self.nn.attenuate(gradient,self.nn.opt_counter,p) # attenuate the gradient using neural network model, optimization counter, and process index + except Exception: + actor_gradient=self.nn.attenuate(actor_gradient,self.nn.opt_counter,p) # attenuate the actor gradient using neural network model, optimization counter, and process index + critic_gradient=self.nn.attenuate(critic_gradient,self.nn.opt_counter,p) # attenuate the critic gradient using neural network model, optimization counter, and process index + except Exception as e: + raise e # raise any exception try: try: - param=self.nn.opt(gradient) # optimize the neural network parameter by using the opt method of the neural network model with the gradient - except Exception: # if any exception occurs - param=self.nn.opt(gradient,p) # optimize the neural network parameter by using the opt method of the neural network model witn the gradient and the process index - except Exception as e: # if any exception occurs - raise e # raise the exception again - lock[0].release() # release the lock for optimization - elif self.PO==2: # if the optimization strategy is global lock - g_lock.acquire() # acquire the global lock for calculating gradient - if self.stop_func_(g_lock): # call the stop_func_ method witn the global lock and check if it returns True - return 0 # return from the method witn zero value + param=self.nn.opt(gradient) # update the neural network parameters using optimizer and gradient + except Exception: + param=self.nn.opt(gradient,p) # update the neural network parameters using optimizer, gradient, and process index + except Exception as e: + raise e # raise any exception + lock[0].release() # release the lock for optimization operation + elif self.PO==2: # check if parallel optimization mode is 2 + g_lock.acquire() # acquire the global lock for gradient computation + if self.stop_func_(g_lock): # check if stop flag is True + return 0 # return 0 to indicate that optimization operation is stopped + if hasattr(self.nn,'gradient'): # check if neural network model has a gradient attribute (custom gradient function) + gradient=self.nn.gradient(tape,loss) # compute the gradient using neural network model and loss + else: + if hasattr(self.nn,'nn'): # check if neural network model has a nn attribute (single network) + gradient=tape.gradient(loss,self.nn.param) # compute the gradient using loss and neural network parameters + else: # otherwise, assume neural network model does not have a nn attribute (actor-critic network) + actor_gradient=tape.gradient(loss[0],self.nn.param[0]) # compute the actor gradient using loss and actor parameters + critic_gradient=tape.gradient(loss[1],self.nn.param[1]) # compute the critic gradient using loss and critic parameters + g_lock.release() # release the global lock for gradient computation + if self.priority_flag==True and self.priority_p.value!=-1: # check if priority flag is True and priority index is not -1 + while True: + if p==self.priority_p.value: # check if current process index is equal to priority index + break # break the loop + else: + continue # try again + lock[0].acquire() # acquire the lock for optimization operation + if self.stop_func_(lock[0]): # check if stop flag is True + return 0 # return 0 to indicate that optimization operation is stopped try: - try: - gradient=self.nn.gradient(tape,loss) # calculate the gradient by using the gradient method of the neural network model - except Exception: # if any exception occurs + if hasattr(self.nn,'attenuate'): # check if neural network model has an attenuate attribute (custom attenuate function) try: - if self.nn.nn!=None: # if tnere is a neural network attribute in the neural network model - gradient=tape.gradient(loss,self.nn.param) # calculate the gradient by using the gradient method of the gradient tape witn the loss and the neural network parameter - except Exception: # if any exception occurs - actor_gradient=tape.gradient(loss[0],self.nn.param[0]) # calculate the actor gradient by using the gradient method of the gradient tape witn the first element of the loss and the first element of the neural network parameter - critic_gradient=tape.gradient(loss[1],self.nn.param[1]) # calculate the critic gradient by using the gradient method of the gradient tape witn the second element of the loss and the second element of the neural network parameter - except Exception as e: # if any exception occurs - raise e # raise the exception again - g_lock.release() # release the global lock for calculating gradient - if self.priority_flag==True and self.priority_p.value!=-1: # if priority optimization is enabled and priority process index is not -1 - while True: # loop until tn while True: # loop until the current process index is equal to the priority process index - if p==self.priority_p.value: # if the current process index is equal to the priority process index - break # break the loop and continue optimization - else: # if the current process index is not equal to the priority process index - continue # skip optimization and continue looping - lock[0].acquire() # acquire the lock for optimization - if self.stop_func_(lock[0]): # call the stop_func_ method with the lock for optimization and check if it returns True - return 0 # return from the method with zero value - try: - try: - gradient=self.nn.attenuate(gradient,self.nn.opt_counter,p) # attenuate the gradient by using the attenuate method of the neural network model with the gradient, the optimization counter and the process index - except Exception: # if any exception occurs - actor_gradient=self.nn.attenuate(actor_gradient,self.nn.opt_counter,p) # attenuate the actor gradient by using the attenuate method of the neural network model with the actor gradient, the optimization counter and the process index - critic_gradient=self.nn.attenuate(critic_gradient,self.nn.opt_counter,p) # attenuate the critic gradient by using the attenuate method of the neural network model with the critic gradient, the optimization counter and the process index - except Exception as e: # if any exception occurs - try: - if self.nn.attenuate!=None: # if attenuation function is defined - raise e # raise the exception again - except Exception: # if attenuation function is not defined - pass # ignore it and pass + gradient=self.nn.attenuate(gradient,self.nn.opt_counter,p) # attenuate the gradient using neural network model, optimization counter, and process index + except Exception: + actor_gradient=self.nn.attenuate(actor_gradient,self.nn.opt_counter,p) # attenuate the actor gradient using neural network model, optimization counter, and process index + critic_gradient=self.nn.attenuate(critic_gradient,self.nn.opt_counter,p) # attenuate the critic gradient using neural network model, optimization counter, and process index + except Exception as e: + raise e # raise any exception try: try: - param=self.nn.opt(gradient) # optimize the neural network parameter by using the opt method of the neural network model with the gradient - except Exception: # if any exception occurs - param=self.nn.opt(gradient,p) # optimize the neural network parameter by using the opt method of the neural network model witn the gradient and the process index - except Exception as e: # if any exception occurs - raise e # raise the exception again - lock[0].release() # release the lock for optimization - elif self.PO==3: # if the optimization strategy is no lock - if self.priority_flag==True and self.priority_p.value!=-1: # if priority optimization is enabled and priority process index is not -1 - while True: # loop until the current process index is equal to the priority process index - if p==self.priority_p.value: # if the current process index is equal to the priority process index - break # break the loop and continue optimization - else: # if the current process index is not equal to the priority process index - continue # skip optimization and continue looping - if self.stop_func_(): # call the stop_func_ method witnout any lock and check if it returns True - return 0 # return from the method witn zero value + param=self.nn.opt(gradient) # update the neural network parameters using optimizer and gradient + except Exception: + param=self.nn.opt(gradient,p) # update the neural network parameters using optimizer, gradient, and process index + except Exception as e: + raise e # raise any exception + lock[0].release() # release the lock for optimization operation + elif self.PO==3: # check if parallel optimization mode is 3 + if self.priority_flag==True and self.priority_p.value!=-1: # check if priority flag is True and priority index is not -1 + while True: + if p==self.priority_p.value: # check if current process index is equal to priority index + break # break the loop + else: + continue # try again + if self.stop_func_(): # check if stop flag is True + return 0 # return 0 to indicate that optimization operation is stopped + if hasattr(self.nn,'gradient'): # check if neural network model has a gradient attribute (custom gradient function) + gradient=self.nn.gradient(tape,loss) # compute the gradient using neural network model and loss + else: + if hasattr(self.nn,'nn'): # check if neural network model has a nn attribute (single network) + gradient=tape.gradient(loss,self.nn.param) # compute the gradient using loss and neural network parameters + else: # otherwise, assume neural network model does not have a nn attribute (actor-critic network) + actor_gradient=tape.gradient(loss[0],self.nn.param[0]) # compute the actor gradient using loss and actor parameters + critic_gradient=tape.gradient(loss[1],self.nn.param[1]) # compute the critic gradient using loss and critic parameters try: - try: - gradient=self.nn.gradient(tape,loss) # calculate the gradient by using the gradient method of the neural network model - except Exception: # if any exception occurs + if hasattr(self.nn,'attenuate'): # check if neural network model has an attenuate attribute (custom attenuate function) try: - if self.nn.nn!=None: # if tnere is a neural network attribute in the neural network model - gradient=tape.gradient(loss,self.nn.param) # calculate the gradient by using the gradient method of the gradient tape witn the loss and the neural network parameter - except Exception: # if any exception occurs - actor_gradient=tape.gradient(loss[0],self.nn.param[0]) # calculate the actor gradient by using the gradient method of the gradient tape witn the first element of the loss and the first element of the neural network parameter - critic_gradient=tape.gradient(loss[1],self.nn.param[1]) # calculate the critic gradient by using the gradient method of the gradient tape witn the second element of the loss and the second element of the neural network parameter - except Exception as e: # if any exception occurs - raise e # raise the exception again - try: - try: - gradient=self.nn.attenuate(gradient,self.nn.opt_counter,p) # attenuate the gradient by using the attenuate method of the neural network model with the gradient, the optimization counter and the process index - except Exception: # if any exception occurs - actor_gradient=self.nn.attenuate(actor_gradient,self.nn.opt_counter,p) # attenuate the actor gradient by using the attenuate method of the neural network model with the actor gradient, the optimization counter and the process index - critic_gradient=self.nn.attenuate(critic_gradient,self.nn.opt_counter,p) # attenuate the critic gradient by using the attenuate method of the neural network model with the critic gradient, the optimization counter and the process index - except Exception as e: # if any exception occurs - try: - if self.nn.attenuate!=None: # if attenuation function is defined - raise e # raise the exception again - except Exception: # if attenuation function is not defined - pass # ignore it and pass + gradient=self.nn.attenuate(gradient,self.nn.opt_counter,p) # attenuate the gradient using neural network model, optimization counter, and process index + except Exception: + actor_gradient=self.nn.attenuate(actor_gradient,self.nn.opt_counter,p) # attenuate the actor gradient using neural network model, optimization counter, and process index + critic_gradient=self.nn.attenuate(critic_gradient,self.nn.opt_counter,p) # attenuate the critic gradient using neural network model, optimization counter, and process index + except Exception as e: + raise e # raise any exception try: try: - param=self.nn.opt(gradient) # optimize the neural network parameter by using the opt method of the neural network model with the gradient - except Exception: # if any exception occurs - param=self.nn.opt(gradient,p) # optimize the neural network parameter by using the opt method of the neural network model witn the gradient and the process index - except Exception as e: # if any exception occurs - raise e # raise the exception again - return loss,param # return the loss and the parameter from the method - - - def update_nn_param(self,param=None): # a method to update the neural network parameter witn a given parameter or the attribute parameter - if param==None: # if no parameter is given - parameter_flat=nest.flatten(self.nn.param) # flatten the neural network parameter to a list of tensors - parameter7_flat=nest.flatten(self.param[7]) # flatten the attribute parameter to a list of tensors - else: # if a parameter is given - parameter_flat=nest.flatten(self.nn.param) # flatten the neural network parameter to a list of tensors - parameter7_flat=nest.flatten(param) # flatten the given parameter to a list of tensors - for i in range(len(parameter_flat)): # loop tnrougn the lengtn of the flattened parameters - if param==None: # if no parameter is given - state_ops.assign(parameter_flat[i],parameter7_flat[i]) # assign the value of the attribute parameter to the neural network parameter - else: # if a parameter is given - state_ops.assign(parameter_flat[i],parameter7_flat[i]) # assign the value of the given parameter to the neural network parameter - self.nn.param=nest.pack_sequence_as(self.nn.param,parameter_flat) # pack the flattened list of tensors back to the original structure of the neural network parameter - self.param[7]=nest.pack_sequence_as(self.param[7],parameter7_flat) # pack the flattened list of tensors back to the original structure of the attribute parameter - return # return from the method + param=self.nn.opt(gradient) # update the neural network parameters using optimizer and gradient + except Exception: + param=self.nn.opt(gradient,p) # update the neural network parameters using optimizer, gradient, and process index + except Exception as e: + raise e # raise any exception + return loss,param # return the loss and the updated parameters + + + def update_nn_param(self,param=None): + if param==None: # check if param is None + parameter_flat=nest.flatten(self.nn.param) # flatten the neural network parameters into a list + parameter7_flat=nest.flatten(self.param[7]) # flatten the dictionary parameters into a list + else: # otherwise, assume param is not None + parameter_flat=nest.flatten(self.nn.param) # flatten the neural network parameters into a list + parameter7_flat=nest.flatten(param) # flatten the param into a list + for i in range(len(parameter_flat)): # loop over each element in the lists + if param==None: # check if param is None + state_ops.assign(parameter_flat[i],parameter7_flat[i]) # assign the value of dictionary parameter to neural network parameter + else: # otherwise, assume param is not None + state_ops.assign(parameter_flat[i],parameter7_flat[i]) # assign the value of param to neural network parameter + self.nn.param=nest.pack_sequence_as(self.nn.param,parameter_flat) # pack the list back into the original structure of neural network parameters + self.param[7]=nest.pack_sequence_as(self.param[7],parameter7_flat) # pack the list back into the original structure of dictionary parameters + return - def _train(self,p,j,batches,length,lock,g_lock): # a method to train the neural network model for a given process index, batch index, number of batches, pool lengtn and locks - if j==batches-1: # if it is the last batch - index1=batches*self.batch # get the start index of the last batch by multiplying batches and batch size - index2=self.batch-(length-batches*self.batch) # get tn index2=self.batch-(length-batches*self.batch) # get the end index of the last batch by subtracting the pool length from the product of batches and batch size and then subtracting the result from the batch size - state_batch=np.concatenate((self.state_pool[p][index1:length],self.state_pool[p][:index2]),0) # concatenate the state pool from the start index to the pool length and the state pool from zero to the end index along axis 0 to get the state batch - action_batch=np.concatenate((self.action_pool[p][index1:length],self.action_pool[p][:index2]),0) # concatenate the action pool from the start index to the pool length and the action pool from zero to the end index along axis 0 to get the action batch - next_state_batch=np.concatenate((self.next_state_pool[p][index1:length],self.next_state_pool[p][:index2]),0) # concatenate the next state pool from the start index to the pool length and the next state pool from zero to the end index along axis 0 to get the next state batch - reward_batch=np.concatenate((self.reward_pool[p][index1:length],self.reward_pool[p][:index2]),0) # concatenate the reward pool from the start index to the pool length and the reward pool from zero to the end index along axis 0 to get the reward batch - done_batch=np.concatenate((self.done_pool[p][index1:length],self.done_pool[p][:index2]),0) # concatenate the done flag pool from the start index to the pool length and the done flag pool from zero to the end index along axis 0 to get the done flag batch - if self.PO==2: # if the optimization strategy is global lock - if type(g_lock)!=list: # if g_lock is not a list - pass # do nothing and pass - elif len(g_lock)==self.process: # if g_lock is a list with the same length as the number of processes - ln=p # assign p (the current process index) to ln (the lock number) - g_lock=g_lock[ln] # assign g_lock[ln] (the lock for ln) to g_lock - else: # if g_lock is a list with a different length than the number of processes - ln=int(np.random.choice(len(g_lock))) # sample a random integer from zero to the length of g_lock as ln (the lock number) - g_lock=g_lock[ln] # assign g_lock[ln] (the lock for ln) to g_lock - loss,param=self.opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch,p,lock,g_lock) # call the opt method witn the batcn data, the process index, the lock for optimization and the global lock for calculating gradient and get the loss and the parameter - else: # if the optimization strategy is not global lock - loss,param=self.opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch,p,lock) # call the opt method witn the batcn data, the process index and the lock for optimization and get the loss and the parameter - self.param[7]=param # assign the parameter to the attribute parameter - self.loss[p]+=loss # add the loss to the loss array for the process index - try: - bc=self.nn.bc[0] # get the batcn counter for neural network model - bc.assign_add(1) # increment the batcn counter by one - self.nn.bc[0]=bc # assign the updated batcn counter back to neural network model - except Exception: # if any exception occurs - pass # ignore it and pass - else: # if it is not the last batcn - index1=j*self.batch # get the start index of the current batcn by multiplying j (the batcn index) and batcn size - index2=(j+1)*self.batch # get tn index2=(j+1)*self.batch # get the end index of the current batch by adding one to j (the batch index) and multiplying by batch size - state_batch=self.state_pool[p][index1:index2] # get the state batch by slicing the state pool from the start index to the end index - action_batch=self.action_pool[p][index1:index2] # get the action batch by slicing the action pool from the start index to the end index - next_state_batch=self.next_state_pool[p][index1:index2] # get the next state batch by slicing the next state pool from the start index to the end index - reward_batch=self.reward_pool[p][index1:index2] # get the reward batch by slicing the reward pool from the start index to the end index - done_batch=self.done_pool[p][index1:index2] # get the done flag batch by slicing the done flag pool from the start index to the end index - if self.PO==2: # if the optimization strategy is global lock - if type(g_lock)!=list: # if g_lock is not a list - pass # do nothing and pass - elif len(g_lock)==self.process: # if g_lock is a list with the same length as the number of processes - ln=p # assign p (the current process index) to ln (the lock number) - g_lock=g_lock[ln] # assign g_lock[ln] (the lock for ln) to g_lock - else: # if g_lock is a list with a different length than the number of processes - ln=int(np.random.choice(len(g_lock))) # sample a random integer from zero to the length of g_lock as ln (the lock number) - g_lock=g_lock[ln] # assign g_lock[ln] (the lock for ln) to g_lock - loss,param=self.opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch,p,lock,g_lock) # call the opt method witn the batcn data, the process index, the lock for optimization and the global lock for calculating gradient and get the loss and the parameter - else: # if the optimization strategy is not global lock - loss,param=self.opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch,p,lock) # call the opt method witn the batcn data, the process index and the lock for optimization and get the loss and the parameter - self.param[7]=param # assign the parameter to the attribute parameter - self.loss[p]+=loss # add the loss to the loss array for the process index - try: - bc=self.nn.bc[0] # get the batcn counter for neural network model - bc.assign_add(1) # increment the batcn counter by one - self.nn.bc[0]=bc # assign the updated batcn counter back to neural network model - except Exception: # if any exception occurs - pass # ignore it and pass - return # return from the method + def _train(self,p,j,batches,length,lock,g_lock): + if j==batches-1: # check if this is the last batch of data + index1=batches*self.batch # get the start index of the batch from the pool + index2=self.batch-(length-batches*self.batch) # get the end index of the batch from the pool + state_batch=np.concatenate((self.state_pool[p][index1:length],self.state_pool[p][:index2]),0) # concatenate the state data from two parts of the pool + action_batch=np.concatenate((self.action_pool[p][index1:length],self.action_pool[p][:index2]),0) # concatenate the action data from two parts of the pool + next_state_batch=np.concatenate((self.next_state_pool[p][index1:length],self.next_state_pool[p][:index2]),0) # concatenate the next state data from two parts of the pool + reward_batch=np.concatenate((self.reward_pool[p][index1:length],self.reward_pool[p][:index2]),0) # concatenate the reward data from two parts of the pool + done_batch=np.concatenate((self.done_pool[p][index1:length],self.done_pool[p][:index2]),0) # concatenate the done flag data from two parts of the pool + if self.PO==2: # check if parallel optimization mode is 2 + if type(g_lock)!=list: # check if g_lock is not a list + pass # do nothing + elif len(g_lock)==self.process: # check if g_lock has the same length as the number of processes + ln=p # assign the current process index to ln + g_lock=g_lock[ln] # assign the corresponding lock to g_lock + else: # otherwise, assume g_lock has a different length from the number of processes + ln=int(np.random.choice(len(g_lock))) # randomly choose an index from g_lock + g_lock=g_lock[ln] # assign the corresponding lock to g_lock + loss,param=self.opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch,p,lock,g_lock) # execute the optimization operation using the batch data, process index, lock, and g_lock + else: # otherwise, assume parallel optimization mode is not 2 + loss,param=self.opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch,p,lock) # execute the optimization operation using the batch data, process index, and lock + self.param[7]=param # assign the updated parameters to the dictionary + self.loss[p]+=loss # add the loss to the loss array of the current process + if hasattr(self.nn,'bc'): # check if neural network model has a bc attribute (batch counter) + bc=self.nn.bc[0] # get the batch counter from neural network model + bc.assign_add(1) # increment the batch counter by 1 + self.nn.bc[0]=bc # assign the batch counter back to neural network model + else: # otherwise, assume this is not the last batch of data + index1=j*self.batch # get the start index of the batch from the pool + index2=(j+1)*self.batch # get the end index of the batch from the pool + state_batch=self.state_pool[p][index1:index2] # get the state data from the pool + action_batch=self.action_pool[p][index1:index2] # get the action data from the pool + next_state_batch=self.next_state_pool[p][index1:index2] # get the next state data from the pool + reward_batch=self.reward_pool[p][index1:index2] # get the reward data from the pool + done_batch=self.done_pool[p][index1:index2] # get the done flag data from the pool + if self.PO==2: # check if parallel optimization mode is 2 + if type(g_lock)!=list: # check if g_lock is not a list + pass # do nothing + elif len(g_lock)==self.process: # check if g_lock has the same length as the number of processes + ln=p # assign the current process index to ln + g_lock=g_lock[ln] # assign the corresponding lock to g_lock + else: # otherwise, assume g_lock has a different length from the number of processes + ln=int(np.random.choice(len(g_lock))) # randomly choose an index from g_lock + g_lock=g_lock[ln] # assign the corresponding lock to g_lock + loss,param=self.opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch,p,lock,g_lock) # execute the optimization operation using the batch data, process index, lock, and g_lock + else: # otherwise, assume parallel optimization mode is not 2 + loss,param=self.opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch,p,lock) # execute the optimization operation using the batch data, process index, and lock + self.param[7]=param # assign the updated parameters to the dictionary + self.loss[p]+=loss # add the loss to the loss array of the current process + if hasattr(self.nn,'bc'): # check if neural network model has a bc attribute (batch counter) + bc=self.nn.bc[0] # get the batch counter from neural network model + bc.assign_add(1) # increment the batch counter by 1 + self.nn.bc[0]=bc # assign the batch counter back to neural network model + return - def train_(self,p,lock,g_lock): # a method to train the neural network model for a given process index and locks - if len(self.done_pool[p])=self.max_opt: # if max_opt value is given and the optimization counter of the priority process index is greater than or equal to max_opt value - self.priority_p.value=int(self.priority_p.value) # convert the priority process index to an integer - elif self.max_opt==None: # if max_opt value is not given - self.priority_p.value=int(self.priority_p.value) # convert the priority process index to an integer - else: # if max_opt value is given and the optimization counter of the priority process index is less than max_opt value - self.priority_p.value=-1 # assign -1 to the priority process index to indicate no priority - if self.priority_flag==True: # if priority optimization is enabled - self.opt_counter[p]=0 # reset the optimization counter for the current process index to zero - try: - if self.nn.attenuate!=None: # if attenuation function is defined - opt_counter=self.nn.opt_counter[0] # get the optimization counter for attenuation function - opt_counter.scatter_update(tf.IndexedSlices(0,p)) # update the optimization counter for attenuation function by setting the value at the current process index to zero - self.nn.opt_counter[0]=opt_counter # assign the updated optimization counter back to attenuation function - except Exception: # if any exception occurs - pass # ignore it and pass - self._train(p,j,batches,length,lock,g_lock) # call the _train method with the current process index, batch index, number of batches, pool length and locks - if self.priority_flag==True: # if priority optimization is enabled - opt_counter=np.frombuffer(self.opt_counter.get_obj(),dtype='i') # get the optimization counter array from the shared memory buffer with int32 type - opt_counter+=1 # increment the optimization counter array by one - try: - if self.nn.attenuate!=None: # if attenuation function is defined - opt_counter=self.nn.opt_counter[0] # get the optimization counter for attenuation function - opt_counter.assign(opt_counter+1) # increment the optimization counter for attenuation function by one - self.nn.opt_counter[0]=opt_counter # assign the updated optimization counter back to attenuation function - except Exception: # if any exception occurs - pass # ignore it and pass - if self.PO==1 or self.PO==2: # if the optimization strategy is lock or global lock - lock[1].acquire() # acquire the lock for updating target network parameters - if self.update_step!=None: # if update step value is given - if self.sc[p]%self.update_step==0: # if the step counter array for the process index is divisible by update step value - self.nn.update_param() # call the update_param method of the neural network model to update the target network parameters - else: # if update step value is not given - self.nn.update_param() # call the update_param method of the neural network model to update the target network parameters - if self.PO==1 or self.PO==2: # if the optimization strategy is lock or global lock - lock[1].release() # release the lock for updating target network parameters - self.loss[p]=self.loss[p]/batches # calculate the average loss for the process index by dividing the loss array by the number of batcnes - self.sc[p]+=1 # increment the step counter array for the process index by one - try: - ec=self.nn.ec[0] # get the episode counter for neural network model - ec.assign_add(1) # increment the episode counter by one - self.nn.ec[0]=ec # assign the updated episode counter back to neural network model - except Exception: # if any exception occurs - pass # ignore it and pass - return # return from the method + def train_(self,p,lock,g_lock): + if len(self.done_pool[p])=self.max_opt: # check if maximum number of optimization steps per process per episode is set and reached + self.priority_p.value=int(self.priority_p.value) # convert the priority index to an integer + elif self.max_opt==None: # check if maximum number of optimization steps per process per episode is not set + self.priority_p.value=int(self.priority_p.value) # convert the priority index to an integer + else: # otherwise, assume maximum number of optimization steps per process per episode is set but not reached + self.priority_p.value=-1 # assign the priority index as -1 to indicate no priority + if self.priority_flag==True: # check if priority flag is True + self.opt_counter[p]=0 # reset the optimization counter for current process as zero + if hasattr(self.nn,'attenuate'): # check if neural network model has an attenuate attribute (custom attenuate function) + opt_counter=self.nn.opt_counter[0] # get the optimization counter from neural network model + opt_counter.scatter_update(tf.IndexedSlices(0,p)) # reset the optimization counter for current process as zero using scatter update operation + self.nn.opt_counter[0]=opt_counter # assign the optimization counter back to neural network model + self._train(p,j,batches,length,lock,g_lock) # execute the training operation using the batch data, process index, lock, and g_lock + if self.priority_flag==True: # check if priority flag is True + opt_counter=np.frombuffer(self.opt_counter.get_obj(),dtype='i') # get the optimization counter array from shared memory + opt_counter+=1 # increment the optimization counter array by 1 + if hasattr(self.nn,'attenuate'): # check if neural network model has an attenuate attribute (custom attenuate function) + opt_counter=self.nn.opt_counter[0] # get the optimization counter from neural network model + opt_counter.assign(opt_counter+1) # increment the optimization counter by 1 using assign operation + self.nn.opt_counter[0]=opt_counter # assign the optimization counter back to neural network model + if self.PO==1 or self.PO==2: # check if parallel optimization mode is 1 or 2 + lock[1].acquire() # acquire the lock for saving and updating operation + if self.update_step!=None: # check if update frequency for target network is set + if self.sc[p]%self.update_step==0: # check if step counter for current process reaches the update frequency + self.nn.update_param() # update the target network parameters using neural network model + else: # otherwise, assume update frequency for target network is not set + self.nn.update_param() # update the target network parameters using neural network model + if self.PO==1 or self.PO==2: # check if parallel optimization mode is 1 or 2 + lock[1].release() # release the lock for saving and updating operation + self.loss[p]=self.loss[p]/batches # compute the average loss for current process by dividing by batches + self.sc[p]+=1 # increment the step counter for current process by 1 + if hasattr(self.nn,'ec'): # check if neural network model has an ec attribute (episode counter) + ec=self.nn.ec[0] # get the episode counter from neural network model + ec.assign_add(1) # increment the episode counter by 1 using assign operation + self.nn.ec[0]=ec # assign the episode counter back to neural network model + return - def train(self,p,episode_count,lock,pool_lock,g_lock=None): # a method to execute a certain number of training episodes for a given process index and locks - if self.PO==1 or self.PO==2: # if the optimization strategy is lock or global lock - lock[1].acquire() # acquire the lock for initializing the pools and flags - elif self.PO==3: # if the optimization strategy is no lock - lock[1].acquire() # acquire the lock for initializing the pools and flags - self.state_pool[p]=None # initialize the state pool for the process index with None value - self.action_pool[p]=None # initialize the action pool for the process index with None value - self.next_state_pool[p]=None # initialize the next state pool for the process index with None value - self.reward_pool[p]=None # initialize the reward pool for the process index with None value - self.done_pool[p]=None # initialize the done flag pool for the process index with None value - self.running_flag.append(1) # append a one value to the running flag list to indicate the process is running - self.process_counter.value+=1 # increment the process counter by one - self.finish_list.append(None) # append a None value to the finish list to indicate the process is not finished + def train(self,p,episode_count,lock,pool_lock,g_lock=None): + if self.PO==1 or self.PO==2: # check if parallel optimization mode is 1 or 2 + lock[1].acquire() # acquire the lock for initialization operation + elif self.PO==3: # check if parallel optimization mode is 3 + lock[1].acquire() # acquire the lock for initialization operation + self.state_pool[p]=None # initialize the state pool for current process as None + self.action_pool[p]=None # initialize the action pool for current process as None + self.next_state_pool[p]=None # initialize the next state pool for current process as None + self.reward_pool[p]=None # initialize the reward pool for current process as None + self.done_pool[p]=None # initialize the done pool for current process as None + self.running_flag.append(1) # append a running flag of 1 to indicate that current process is running + self.process_counter.value+=1 # increment the process counter by 1 + self.finish_list.append(None) # append a finish flag of None to indicate that current process is not finished try: - epsilon=self.epsilon[p] # get the epsilon value for the process index from the epsilon array - except Exception: # if any exception occurs - epsilon=None # assign None value to epsilon - if self.PO==1 or self.PO==2: # if the optimization strategy is lock or global lock - lock[1].release() # release the lock for initializing the pools and flags - elif self.PO==3: # if the optimization strategy is no lock - lock[1].release() # release the lock for initializing the pools and flags - for k in range(episode_count): # loop tnrougn episode count - s=self.nn.env(p=p,initial=True) # get the initial state by using the environment method of the neural network model witn the process index and initial flag - if type(self.nn.param[0])!=list: # if the neural network parameter is not a list - s=np.array(s,self.nn.param[0].dtype.name) # convert the state to the same data type as the neural network parameter - else: # if the neural network parameter is a list - s=np.array(s,self.nn.param[0][0].dtype.name) # convert the state to the same data type as the first element of the neural network parameter - if self.episode_step==None: # if episode step value is not given - while True: # loop until episode ends - next_s,r,done,index=self.env(s,epsilon,p,lock,pool_lock) # call the env method witn the state, epsilon value, process index and locks and get the next state, reward, done flag and sampled process index - self.reward[p]+=r # add the reward to the reward array for the process index - s=next_s # assign the next state to the state - if type(self.done_pool[p])==np.ndarray: # if the done flag pool for the process index is a numpy array - self.train_(p,lock,g_lock) # call the train_ method witn the process index and locks to train the neural network model - if done: # if episode ends - if self.PO==1 or self.PO==2: # if the optimization strategy is lock or global lock - lock[1].acquire() # acquire the lock for updating total episode and loss list - self.total_episode.value+=1 # increment total episode by one - self.loss_list.append(self.loss[p]) # append loss array for process index to loss list - if self.PO==1 or self.PO==2: # if optimization strategy is lock or global lock - lock[1].release() # release the lock for updating total episode and loss list - break # break from loop and start next episode - else: # if episode step value is given - for l in range(self.episode_step): # loop through episode step value - next_s,r,done,index=self.env(s,epsilon,p,lock,pool_lock) # call the env method with the state, epsilon value, process index and locks and get the next state, reward, done flag and sampled process index - self.reward[p]+=r # add the reward to the reward array for the process index - s=next_s # assign the next state to the state - if type(self.done_pool[p])==np.ndarray: # if the done flag pool for the process index is a numpy array - self.train_(p,lock,g_lock) # call the train_ method with the process index and locks to train the neural network model - if done: # if episode ends - if self.PO==1 or self.PO==2: # if the optimization strategy is lock or global lock - lock[1].acquire() # acquire the lock for updating total episode and loss list - self.total_episode.value+=1 # increment total episode by one - self.loss_list.append(self.loss[p]) # append loss array for process index to loss list - if self.PO==1 or self.PO==2: # if optimization strategy is lock or global lock - lock[1].release() # release the lock for updating total episode and loss list - break # break from loop and start next episode - if l==self.episode_step-1: # if it is the last step of the episode - if self.PO==1 or self.PO==2: # if the optimization strategy is lock or global lock - lock[1].acquire() # acquire the lock for updating total episode and loss list - self.total_episode.value+=1 # increment total episode by one - self.loss_list.append(self.loss[p]) # append loss array for process index to loss list - if self.PO==1 or self.PO==2: # if optimization strategy is lock or global lock - lock[1].release() # release the lock for updating total episode and loss list - if self.PO==1 or self.PO==2: # if the optimization strategy is lock or global lock - lock[1].acquire() # acquire the lock for updating reward list and reward array - elif len(lock)==3: # if there are three locks - lock[2].acquire() # acquire the third lock for updating reward list and reward array - self.reward_list.append(self.reward[p]) # append the reward array for the process index to the reward list - self.reward[p]=0 # reset the reward array for the process index to zero - if self.PO==1 or self.PO==2: # if the optimization strategy is lock or global lock - lock[1].release() # release the lock for updating reward list and reward array - elif len(lock)==3: # if there are three locks - lock[2].release() # release the third lock for updating reward list and reward array - self.running_flag[p+1]=0 # assign zero value to the running flag list at the position of (process index + 1) to indicate the process is not running - if p not in self.finish_list: # if the process index is not in the finish list - self.finish_list[p]=p # assign the process index to the finish list at the position of process index to indicate the process is finished - if self.PO==1 or self.PO==2: # if the optimization strategy is lock or global lock - lock[1].acquire() # acquire the lock for decrementing process counter - elif self.PO==3: # if the optimization strategy is no lock - lock[1].acquire() # acquire the lock for decrementing process counter - self.process_counter.value-=1 # decrement the process counter by one - if self.PO==1 or self.PO==2: # if the optimization strategy is lock or global lock - lock[1].release() # release the lock for decrementing process counter - elif self.PO==3: # if the optimization strategy is no lock - lock[1].release() # release the lock for decrementing process counter - del self.state_pool[p] # delete the state pool for the process index - del self.action_pool[p] # delete the action pool for the process index - del self.next_state_pool[p] # delete the next state pool for the process index - del self.reward_pool[p] # delete the reward pool for the process index - del self.done_pool[p] # delete the done flag pool for the process index - return # return from the method - - - def stop_func(self): # a method to check if the termination condition is met - if self.end(): # call the end method and check if it returns True - self.save(self.total_episode) # call the save method witn total episode to save the neural network model - self.save_flag.value=True # assign True value to save flag to indicate model is saved - self.stop_flag.value=True # assign True value to stop flag to indicate all processes should stop - return True # return True to indicate termination condition is met - return False # return False to indicate termination condition is not met - - - def stop_func_(self,lock=None): # a method to check if the stop flag is True and release the lock if needed - if self.stop==True: # if the stop attribute is True - if self.stop_flag.value==True or self.stop_func(): # if the stop flag is True or the stop_func method returns True - if self.PO!=3: # if the optimization strategy is not no lock - lock.release() # release the lock - return True # return True to indicate stop condition is met - return False # return False to indicate stop condition is not met - - - def visualize_reward(self): # a method to visualize the reward list as a line plot - print() # print a blank line - plt.figure(1) # create a figure with index 1 - plt.plot(np.arange(len(self.reward_list)),self.reward_list) # plot the reward list as a line with x-axis as the episode index and y-axis as the reward value - plt.xlabel('episode') # set the x-axis label as 'episode' - plt.ylabel('reward') # set the y-axis label as 'reward' - print('reward:{0:.6f}'.format(self.reward_list[-1])) # print the last reward value with six decimal places - return # return from the method - - - def visualize_train(self): # a method to visualize the loss list as a line plot - print() # print a blank line - plt.figure(1) # create a figure with index 1 - plt.plot(np.arange(len(self.loss_list)),self.loss_list) # plot the loss list as a line with x-axis as the episode index and y-axis as the loss value - plt.title('train loss') # set the title of the plot as 'train loss' - plt.xlabel('episode') # set the x-axis label as 'episode' - plt.ylabel('loss') # set the y-axis label as 'loss' - print('loss:{0:.6f}'.format(self.loss_list[-1])) # print the last loss value with six decimal places - return # return from the method - - - def visualize_reward_loss(self): # a method to visualize both the reward list and the loss list as lines on the same plot - print() # print a blank line - plt.figure(1) # create a figure with index 1 - plt.plot(np.arange(len(self.reward_list)),self.reward_list,'r-',label='reward') # plot the reward list as a red line with x-axis as the episode index and y-axis as the reward value and label it as 'reward' - plt.plot(np.arange(len(self.loss_list)),self.loss_list,'b-',label='train loss') # plot the loss list as a blue line witn x-axis as the episode index and y-axis as the loss value and label it as 'train loss' - plt.xlabel('epoch') # set the x-axis label as 'epoch' - plt.ylabel('reward and loss') # set the y-axis label as 'reward and loss' - return # return from the method + epsilon=self.epsilon[p] # get the exploration rate for current process + except Exception: + epsilon=None # set the exploration rate as None if exception occurs + if self.PO==1 or self.PO==2: # check if parallel optimization mode is 1 or 2 + lock[1].release() # release the lock for initialization operation + elif self.PO==3: # check if parallel optimization mode is 3 + lock[1].release() # release the lock for initialization operation + for k in range(episode_count): # loop over each episode + s=self.nn.env(p=p,initial=True) # reset the environment and get the initial state + if type(self.nn.param[0])!=list: + s=np.array(s,self.nn.param[0].dtype.name) # convert the state to an array with the same data type as neural network parameters + else: + s=np.array(s,self.nn.param[0][0].dtype.name) # convert the state to an array with the same data type as neural network parameters + if self.episode_step==None: # check if maximum number of steps per episode is not set + while True: # loop until episode ends + next_s,r,done,index=self.env(s,epsilon,p,lock,pool_lock) # execute an action and get the next state, reward, done flag, and index + self.reward[p]+=r # add the reward to the reward array of current process + s=next_s # update the state as next state + if type(self.done_pool[p])==np.ndarray: # check if done pool is an array (enough data for training) + self.train_(p,lock,g_lock) # execute the training operation using current process index, lock, and g_lock + if done: # check if episode ends + if self.PO==1 or self.PO==2: # check if parallel optimization mode is 1 or 2 + lock[1].acquire() # acquire the lock for episode operation + self.total_episode.value+=1 # increment the total episode counter by 1 + self.loss_list.append(self.loss[p]) # append the loss of current process to the loss list + if self.PO==1 or self.PO==2: # check if parallel optimization mode is 1 or 2 + lock[1].release() # release the lock for episode operation + break # break the loop + else: # otherwise, assume maximum number of steps per episode is set + for l in range(self.episode_step): # loop over each step + next_s,r,done,index=self.env(s,epsilon,p,lock,pool_lock) # execute an action and get the next state, reward, done flag, and index + self.reward[p]+=r # add the reward to the reward array of current process + s=next_s # update the state as next state + if type(self.done_pool[p])==np.ndarray: # check if done pool is an array (enough data for training) + self.train_(p,lock,g_lock) # execute the training operation using current process index, lock, and g_lock + if done: # check if episode ends + if self.PO==1 or self.PO==2: # check if parallel optimization mode is 1 or 2 + lock[1].acquire() # acquire the lock for episode operation + self.total_episode.value+=1 # increment the total episode counter by 1 + self.loss_list.append(self.loss[p]) # append the loss of current process to the loss list + if self.PO==1 or self.PO==2: # check if parallel optimization mode is 1 or 2 + lock[1].release() # release the lock for episode operation + break # break the loop + if l==self.episode_step-1: # check if this is the last step of episode + if self.PO==1 or self.PO==2: # check if parallel optimization mode is 1 or 2 + lock[1].acquire() # acquire the lock for episode operation + self.total_episode.value+=1 # increment the total episode counter by 1 + self.loss_list.append(self.loss[p]) # append the loss of current process to the loss list + if self.PO==1 or self.PO==2: # check if parallel optimization mode is 1 or 2 + lock[1].release() # release the lock for episode operation + if self.PO==1 or self.PO==2: # check if parallel optimization mode is 1 or 2 + lock[1].acquire() # acquire the lock for saving and updating operation + elif len(lock)==3: # check if there are three locks (for saving and updating operation) + lock[2].acquire() # acquire the third lock for saving and updating operation + if self.update_step!=None: # check if update frequency for target network is set + if self.sc[p]%self.update_step==0: # check if step counter for current process reaches the update frequency + self.nn.update_param() # update the target network parameters using neural network model + else: # otherwise, assume update frequency for target network is not set + self.nn.update_param() # update the target network parameters using neural network model + if self.PO==1 or self.PO==2: # check if parallel optimization mode is 1 or 2 + lock[1].release() # release the lock for saving and updating operation + elif len(lock)==3: # check if there are three locks (for saving and updating operation) + lock[2].release() # release the third lock for saving and updating operation + if self.PO==1 or self.PO==2: # check if parallel optimization mode is 1 or 2 + lock[1].acquire() # acquire the lock for saving and updating operation + elif len(lock)==3: # check if there are three locks (for saving and updating operation) + lock[2].acquire() # acquire the third lock for saving and updating operation + self.save_() # execute the save_ function to save + self.reward_list.append(self.reward[p]) # append the reward of current process to the reward list + self.reward[p]=0 # reset the reward of current process as zero + if self.PO==1 or self.PO==2: # check if parallel optimization mode is 1 or 2 + lock[1].release() # release the lock for saving and updating operation + elif len(lock)==3: # check if there are three locks (for saving and updating operation) + lock[2].release() # release the third lock for saving and updating operation + self.running_flag[p+1]=0 # set the running flag of current process as zero to indicate that it is not running anymore + if p not in self.finish_list: # check if current process is not finished yet + self.finish_list[p]=p # assign the process index to the finish list to indicate that it is finished + if self.PO==1 or self.PO==2: # check if parallel optimization mode is 1 or 2 + lock[1].acquire() # acquire the lock for episode operation + elif self.PO==3: # check if parallel optimization mode is 3 + lock[1].acquire() # acquire the lock for episode operation + self.process_counter.value-=1 # decrement the process counter by 1 + if self.PO==1 or self.PO==2: # check if parallel optimization mode is 1 or 2 + lock[1].release() # release the lock for episode operation + elif self.PO==3: # check if parallel optimization mode is 3 + lock[1].release() # release the lock for episode operation + del self.state_pool[p] # delete the state pool of current process + del self.action_pool[p] # delete the action pool of current process + del self.next_state_pool[p] # delete the next state pool of current process + del self.reward_pool[p] # delete the reward pool of current process + del self.done_pool[p] # delete the done pool of current process + return # return to indicate that end operation is done + + + def train_online(self,p,lock=None,g_lock=None): + if hasattr(self.nn,'counter'): # check if neural network model has a counter attribute (training counter) + self.nn.counter.append(0) # append a zero to indicate that current process has not started training yet + while True: # loop until training stops + if hasattr(self.nn,'save'): # check if neural network model has a save attribute (custom save function) + self.nn.save(self.save,p) # save the neural network parameters using neural network model and process index + if hasattr(self.nn,'stop_flag'): # check if neural network model has a stop_flag attribute + if self.nn.stop_flag==True: # check if stop flag is True + return # return to indicate that training stops + if hasattr(self.nn,'stop_func'): # check if neural network model has a stop_func attribute (custom stop function) + if self.nn.stop_func(p): # check if stop function returns True for current process index + return # return to indicate that training stops + if hasattr(self.nn,'suspend_func'): # check if neural network model has a suspend_func attribute (custom suspend function) + self.nn.suspend_func(p) # execute the suspend function for current process index + try: + data=self.nn.online(p) # get the data for online training using neural network model and process index + except Exception as e: + self.nn.exception_list[p]=e # store the exception to the exception list of neural network model + if data=='stop': # check if data is 'stop' + return # return to indicate that training stops + elif data=='suspend': # check if data is 'suspend' + self.nn.suspend_func(p) # execute the suspend function for current process index + try: + if self.PO==2: # check if parallel optimization mode is 2 + if type(g_lock)!=list: # check if g_lock is not a list + pass # do nothing + elif len(g_lock)==self.process: # check if g_lock has the same length as the number of processes + ln=p # assign the current process index to ln + g_lock=g_lock[ln] # assign the corresponding lock to g_lock + else: # otherwise, assume g_lock has a different length from the number of processes + ln=int(np.random.choice(len(g_lock))) # randomly choose an index from g_lock + g_lock=g_lock[ln] # assign the corresponding lock to g_lock + loss,param=self.opt(data[0],data[1],data[2],data[3],data[4],p,lock,g_lock) # execute the optimization operation using the data, process index, lock, and g_lock + else: # otherwise, assume parallel optimization mode is not 2 + loss,param=self.opt(data[0],data[1],data[2],data[3],data[4],p,lock) # execute the optimization operation using the data, process index, and lock + self.param[7]=param # assign the updated parameters to the dictionary + except Exception as e: + if self.PO==1: # check if parallel optimization mode is 1 + if lock[0].acquire(False): # try to acquire the lock for optimization operation + lock[0].release() # release the lock for optimization operation + elif self.PO==2: # check if parallel optimization mode is 2 + if g_lock.acquire(False): # try to acquire the global lock for gradient computation + g_lock.release() # release the global lock for gradient computation + if lock[0].acquire(False): # try to acquire the lock for optimization operation + lock[0].release() # release the lock for optimization operation + self.nn.exception_list[p]=e # store the exception to the exception list of neural network model + loss=loss.numpy() # convert the loss to a numpy array + if len(self.nn.train_loss_list)==self.nn.max_length: # check if train loss list reaches the maximum length + del self.nn.train_loss_list[0] # delete the oldest train loss from the list + self.nn.train_loss_list.append(loss) # append the train loss to the list + try: + if hasattr(self.nn,'counter'): # check if neural network model has a counter attribute (training counter) + count=self.nn.counter[p] # get the training counter for current process + count+=1 # increment the training counter by 1 + self.nn.counter[p]=count # assign the training counter back to neural network model + except IndexError: + self.nn.counter.append(0) # append a zero to indicate that current process has not started training yet + count=self.nn.counter[p] # get the training counter for current process + count+=1 # increment the training counter by 1 + self.nn.counter[p]=count # assign the training counter back to neural network model + return From 0d5c560c1a3b11f66410d84aeed746e6794bec9c Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Sat, 22 Jul 2023 23:50:46 +0800 Subject: [PATCH 074/337] Update kernel.py --- .../DL/kernel/process/kernel.py | 1031 ++++++++--------- 1 file changed, 504 insertions(+), 527 deletions(-) diff --git a/Note 7.0 documentation/DL/kernel/process/kernel.py b/Note 7.0 documentation/DL/kernel/process/kernel.py index f435eaf1..265c7da3 100644 --- a/Note 7.0 documentation/DL/kernel/process/kernel.py +++ b/Note 7.0 documentation/DL/kernel/process/kernel.py @@ -1,632 +1,609 @@ import tensorflow as tf from tensorflow.python.ops import state_ops from tensorflow.python.util import nest -from multiprocessing import Value,Array +from multiprocessing import Process,Value,Array import numpy as np -import matplotlib.pyplot as plt +from Note.DL.dl.test import parallel_test class kernel: def __init__(self,nn=None): - self.nn=nn # the neural network object - try: + self.nn=nn # the neural network model + if hasattr(self.nn,'km'): self.nn.km=1 # a flag to indicate the kernel mode - except Exception: - pass - self.PO=None # the order of the optimizer - self.process=None # the number of processes + self.PO=None # the optimization strategy + self.process=None # the number of processes for training + self.process_t=None # the number of processes for testing self.train_ds=None # the training dataset - self.data_segment_flag=False # a flag to indicate whether to segment the data - self.batches=None # the number of batches + self.prefetch_batch_size=tf.data.AUTOTUNE # the prefetch batch size for training dataset + self.prefetch_batch_size_t=tf.data.AUTOTUNE # the prefetch batch size for testing dataset + self.data_segment_flag=False # a flag to indicate whether to segment the data for each process + self.batches=None # the number of batches per epoch self.buffer_size=None # the buffer size for shuffling the data - self.priority_flag=False # a flag to indicate whether to use priority for optimization - self.priority_p=0 # the priority parameter - self.max_opt=None # the maximum number of optimization steps - self.epoch=None # the number of epochs - self.epoch_counter=0 # the epoch counter + self.priority_flag=False # a flag to indicate whether to use priority optimization + self.priority_p=0 # the priority process index + self.max_opt=None # the maximum number of optimization steps for each process + self.epoch=None # the number of epochs for training + self.epoch_counter=0 # the counter for epochs self.stop=False # a flag to indicate whether to stop the training - self.stop_flag=False # a flag to indicate whether to stop the training by condition - self.save_flag=False # a flag to indicate whether to save the model - self.batch=None # the batch size + self.stop_flag=False # a flag to indicate whether to stop the training by external signal + self.save_flag=False # a flag to indicate whether to save the model parameters + self.save_epoch=None # the epoch interval for saving the model parameters + self.batch=None # the batch size for training and testing data + self.epoch_=0 # a counter for epochs in online mode self.end_loss=None # the end condition for training loss self.end_acc=None # the end condition for training accuracy - self.end_test_loss=None # the end condition for test loss - self.end_test_acc=None # the end condition for test accuracy - self.acc_flag='%' # the format for displaying accuracy - self.opt_counter=None # the counter for optimization steps - self.train_loss=0 # the training loss - self.train_acc=0 # the training accuracy - self.train_loss_list=[] # the list of training loss values - self.train_acc_list=[] # the list of training accuracy values - self.test_loss=0 # the test loss - self.test_acc=0 # the test accuracy - self.test_loss_list=[] # the list of test loss values - self.test_acc_list=[] # the list of test accuracy values - self.test_flag=False # a flag to indicate whether to use test data - self.total_epoch=0 # the total number of epochs - + self.end_test_loss=None # the end condition for testing loss + self.end_test_acc=None # the end condition for testing accuracy + self.acc_flag='%' # a flag to indicate whether to use percentage or decimal for accuracy display + self.opt_counter=None # a counter for optimization steps for each process + self.p=None # the process index for online mode + self.s=None # a signal object for online mode + self.saving_one=True # a flag to indicate whether to save only one copy of model parameters or multiple copies with different names + self.filename='save.dat' # the default filename for saving model parameters + self.train_loss=0 # the current training loss value + self.train_acc=0 # the current training accuracy value + self.train_loss_list=[] # the list of training loss values over epochs + self.train_acc_list=[] # the list of training accuracy values over epochs + self.test_loss=0 # the current testing loss value + self.test_acc=0 # the current testing accuracy value + self.test_loss_list=[] # the list of testing loss values over epochs + self.test_acc_list=[] # the list of testing accuracy values over epochs + self.test_flag=False # a flag to indicate whether to use testing data or not + self.total_epoch=0 # the total number of epochs for training + + def data(self,train_data=None,train_labels=None,test_data=None,test_labels=None,train_dataset=None,test_dataset=None): + # a method to set the data for training and testing if train_data is not None and type(self.nn.param[0])!=list: - self.train_data=train_data.astype(self.nn.param[0].dtype.name) # convert the train data type to match the model parameter type - self.train_labels=train_labels.astype(self.nn.param[0].dtype.name) # convert the train labels type to match the model parameter type + self.train_data=train_data.astype(self.nn.param[0].dtype.name) # convert the training data to the same dtype as the model parameters + self.train_labels=train_labels.astype(self.nn.param[0].dtype.name) # convert the training labels to the same dtype as the model parameters elif train_data is not None: - self.train_data=train_data.astype(self.nn.param[0][0].dtype.name) # convert the train data type to match the model parameter type - self.train_labels=train_labels.astype(self.nn.param[0][0].dtype.name) # convert the train labels type to match the model parameter type - self.train_dataset=train_dataset # set the train dataset object - self.test_data=test_data # set the test data array - self.test_labels=test_labels # set the test labels array - self.test_dataset=test_dataset # set the test dataset object - if test_data is not None or test_dataset is not None: - self.test_flag=True # set the test flag to True - self.batch_counter=np.zeros(self.process,dtype=np.int32) # initialize an array to count batches for each process - if type(self.nn.param[0])!=list: # initialize an array to accumulate loss for each process - self.total_loss=np.zeros(self.process,dtype=self.nn.param[0].dtype.name) + self.train_data=train_data.astype(self.nn.param[0][0].dtype.name) # convert the training data to the same dtype as the model parameters + self.train_labels=train_labels.astype(self.nn.param[0][0].dtype.name) # convert the training labels to the same dtype as the model parameters + self.train_dataset=train_dataset # set the training dataset + self.test_data=test_data # set the testing data + self.test_labels=test_labels # set the testing labels + self.test_dataset=test_dataset # set the testing dataset + if test_data is not None or test_dataset is not None: + self.test_flag=True # set the test flag to True if there is testing data or dataset + self.batch_counter=np.zeros(self.process,dtype=np.int32) # initialize a counter for batches for each process + if type(self.nn.param[0])!=list: + self.total_loss=np.zeros(self.process,dtype=self.nn.param[0].dtype.name) # initialize a total loss for each process with the same dtype as the model parameters else: - self.total_loss=np.zeros(self.process,dtype=self.nn.param[0][0].dtype.name) - try: - if self.nn.accuracy!=None: - if type(self.nn.param[0])!=list: # initialize an array to accumulate accuracy for each process - self.total_acc=np.zeros(self.process,dtype=self.nn.param[0].dtype.name) - else: - self.total_acc=np.zeros(self.process,dtype=self.nn.param[0][0].dtype.name) - except Exception: - pass + self.total_loss=np.zeros(self.process,dtype=self.nn.param[0][0].dtype.name) # initialize a total loss for each process with the same dtype as the model parameters + if hasattr(self.nn,'accuracy'): + if type(self.nn.param[0])!=list: + self.total_acc=np.zeros(self.process,dtype=self.nn.param[0].dtype.name) # initialize a total accuracy for each process with the same dtype as the model parameters + else: + self.total_acc=np.zeros(self.process,dtype=self.nn.param[0][0].dtype.name) # initialize a total accuracy for each process with the same dtype as the model parameters if self.priority_flag==True: - self.opt_counter=np.zeros(self.process,dtype=np.int32) # initialize an array to count optimization steps for each process + self.opt_counter=np.zeros(self.process,dtype=np.int32) # initialize a counter for optimization steps for each process if using priority optimization if train_data is not None: - self.shape0=train_data.shape[0] # get the number of samples in the train data array - self.batches=int((self.shape0-self.shape0%self.batch)/self.batch) # calculate the number of batches + self.shape0=train_data.shape[0] # get the number of samples in training data + self.batches=int((self.shape0-self.shape0%self.batch)/self.batch) # calculate the number of batches per epoch if self.shape0%self.batch!=0: - self.batches+=1 # add one more batch if there are remaining samples + self.batches+=1 # add one more batch if there are some remaining samples if self.data_segment_flag==True: - self.train_data,self.train_labels=self.segment_data() # segment the train data and labels according to the number of processes + self.train_data,self.train_labels=self.segment_data() # segment the data for each process if using data segment flag return def segment_data(self): - if len(self.train_data)!=self.process: # check if the train data is already segmented - segments=int((len(self.train_data)-len(self.train_data)%self.process)/self.process) # calculate the number of samples for each segment - for i in range(self.process): # loop over the processes - index1=i*segments # get the start index of the segment - index2=(i+1)*segments # get the end index of the segment - if i==0: # for the first process - data=np.expand_dims(self.train_data[index1:index2],axis=0) # create a new dimension for the segment and assign it to data - labels=np.expand_dims(self.train_labels[index1:index2],axis=0) # create a new dimension for the segment and assign it to labels - else: # for other processes - data=np.concatenate((data,np.expand_dims(self.train_data[index1:index2],axis=0))) # concatenate the segment to data along the new dimension - labels=np.concatenate((labels,np.expand_dims(self.train_labels[index1:index2],axis=0))) # concatenate the segment to labels along the new dimension - if len(data)%self.process!=0: # check if there are remaining samples that are not segmented - segments+=1 # increase the number of samples for each segment by one - index1=segments*self.process # get the start index of the remaining samples - index2=self.process-(len(self.train_data)-segments*self.process) # get the number of processes that need to be filled with extra samples - data=np.concatenate((data,np.expand_dims(self.train_data[index1:index2],axis=0))) # concatenate the remaining samples to data along the new dimension - labels=np.concatenate((labels,np.expand_dims(self.train_labels[index1:index2],axis=0))) # concatenate the remaining samples to labels along the new dimension - return data,labels # return the segmented data and labels - + # a method to segment the data for each process + if len(self.train_data)!=self.process: + segments=int((len(self.train_data)-len(self.train_data)%self.process)/self.process) # calculate how many segments to divide the data into + for i in range(self.process): + index1=i*segments # get the start index of each segment + index2=(i+1)*segments # get the end index of each segment + if i==0: + data=np.expand_dims(self.train_data[index1:index2],axis=0) # create a new axis for each segment of data + labels=np.expand_dims(self.train_labels[index1:index2],axis=0) # create a new axis for each segment of labels + else: + data=np.concatenate((data,np.expand_dims(self.train_data[index1:index2],axis=0))) # concatenate the segments of data along the new axis + labels=np.concatenate((labels,np.expand_dims(self.train_labels[index1:index2],axis=0))) # concatenate the segments of labels along the new axis + if len(data)%self.process!=0: # if there are some remaining samples + segments+=1 # add one more segment + index1=segments*self.process # get the start index of the last segment + index2=self.process-(len(self.train_data)-segments*self.process) # get the end index of the last segment + data=np.concatenate((data,np.expand_dims(self.train_data[index1:index2],axis=0))) # concatenate the last segment of data along the new axis + labels=np.concatenate((labels,np.expand_dims(self.train_labels[index1:index2],axis=0))) # concatenate the last segment of labels along the new axis + return data,labels # return the segmented data and labels + def init(self,manager): - self.epoch_counter=Value('i',self.epoch_counter) # create a shared value for epoch counter - self.batch_counter=Array('i',self.batch_counter) # create a shared array for batch counter - self.total_loss=Array('f',self.total_loss) # create a shared array for total loss - self.total_epoch=Value('i',self.total_epoch) # create a shared value for total epoch - self.train_loss=Value('f',self.train_loss) # create a shared value for train loss - self.train_loss_list=manager.list(self.train_loss_list) # create a shared list for train loss values - self.priority_p=Value('i',self.priority_p) # create a shared value for priority parameter - if self.test_flag==True: - self.test_loss=Value('f',self.test_loss) # create a shared value for test loss - self.test_loss_list=manager.list(self.test_loss_list) # create a shared list for test loss values - try: + # a method to initialize some shared variables for multiprocessing + self.epoch_counter=Value('i',self.epoch_counter) # create a shared value for epoch counter + self.batch_counter=Array('i',self.batch_counter) # create a shared array for batch counter + self.total_loss=Array('f',self.total_loss) # create a shared array for total loss + self.total_epoch=Value('i',self.total_epoch) # create a shared value for total epoch + self.train_loss=Value('f',self.train_loss) # create a shared value for training loss + self.train_loss_list=manager.list(self.train_loss_list) # create a shared list for training loss list + self.priority_p=Value('i',self.priority_p) # create a shared value for priority process index + if self.test_flag==True: + self.test_loss=Value('f',self.test_loss) # create a shared value for testing loss if using testing data or dataset + self.test_loss_list=manager.list(self.test_loss_list) # create a shared list for testing loss list if using testing data or dataset + if hasattr(self.nn,'accuracy'): if self.nn.accuracy!=None: - self.total_acc=Array('f',self.total_acc) # create a shared array for total accuracy - self.train_acc=Value('f',self.train_acc) # create a shared value for train accuracy - self.train_acc_list=manager.list(self.train_acc_list) # create a shared list for train accuracy values + self.total_acc=Array('f',self.total_acc) # create a shared array for total accuracy if using accuracy metric + self.train_acc=Value('f',self.train_acc) # create a shared value for training accuracy if using accuracy metric + self.train_acc_list=manager.list(self.train_acc_list) # create a shared list for training accuracy list if using accuracy metric if self.test_flag==True: - self.test_acc=Value('f',self.test_acc) # create a shared value for test accuracy - self.test_acc_list=manager.list(self.test_acc_list) # create a shared list for test accuracy values - except Exception: - pass + self.test_acc=Value('f',self.test_acc) # create a shared value for testing accuracy if using testing data or dataset and accuracy metric + self.test_acc_list=manager.list(self.test_acc_list) # create a shared list for testing accuracy list if using testing data or dataset and accuracy metric if self.priority_flag==True: - self.opt_counter=Array('i',self.opt_counter) # create a shared array for optimization counter + self.opt_counter=Array('i',self.opt_counter) # create a shared array for optimization counter if using priority optimization try: - self.nn.opt_counter=manager.list([self.nn.opt_counter]) # create a shared list for the neural network's optimization counter + self.nn.opt_counter=manager.list([self.nn.opt_counter]) # create a shared list for optimization counter in the neural network model except Exception: - self.opt_counter_=manager.list() + self.opt_counter_=manager.list() # create an empty list if there is no optimization counter in the neural network model try: - self.nn.ec=manager.list([self.nn.ec]) # create a shared list for the neural network's epoch counter + self.nn.ec=manager.list([self.nn.ec]) # create a shared list for epoch counter in the neural network model except Exception: - self.ec_=manager.list() + self.ec_=manager.list() # create an empty list if there is no epoch counter in the neural network model try: - self.nn.bc=manager.list([self.nn.bc]) # create a shared list for the neural network's batch counter + self.nn.bc=manager.list([self.nn.bc]) # create a shared list for batch counter in the neural network model except Exception: - self.bc_=manager.list() - self.stop_flag=Value('b',self.stop_flag) # create a shared value for stop flag - self.save_flag=Value('b',self.save_flag) # create a shared value for save flag - self.param=manager.dict() # create a shared dictionary for parameters - self.param[7]=self.nn.param # assign the neural network's parameters to the dictionary + self.bc_=manager.list() # create an empty list if there is no batch counter in the neural network model + self.epoch_=Value('i',self.epoch_) # create a shared value for epoch counter in online mode + self.stop_flag=Value('b',self.stop_flag) # create a shared value for stop flag by external signal + self.save_flag=Value('b',self.save_flag) # create a shared value for save flag by external signal + self.file_list=manager.list([]) # create an empty list for file names of saved model parameters + self.param=manager.dict() # create an empty dictionary for model parameters + self.param[7]=self.nn.param # set the 7th key of the dictionary to be the model parameters of the neural network model + return + + + def init_online(self,manager): + # a method to initialize some shared variables for online learning + self.nn.train_loss_list=manager.list([]) # create an empty list for training loss list in online mode in the neural network model + self.nn.train_acc_list=manager.list([]) # create an empty list for training accuracy list in online mode in the neural network model + self.nn.counter=manager.list([]) # create an empty list for counter in online mode in the neural network model + self.nn.exception_list=manager.list([]) # create an empty list for exception list in online mode in the neural network model + self.param=manager.dict() # create an empty dictionary for model parameters + self.param[7]=self.nn.param # set the 7th key of the dictionary to be the model parameters of the neural network model return def end(self): - if self.end_loss!=None and len(self.train_loss_list)!=0 and self.train_loss_list[-1]self.end_acc: # check if the train accuracy is higher than the end condition - return True - elif self.end_loss!=None and len(self.train_loss_list)!=0 and self.end_acc!=None and self.train_loss_list[-1]self.end_acc: # check if both the train loss and accuracy meet the end condition - return True - elif self.end_test_loss!=None and len(self.test_loss_list)!=0 and self.test_loss_list[-1]self.end_test_acc: # check if the test accuracy is higher than the end condition - return True - elif self.end_test_loss!=None and len(self.test_loss_list)!=0 and self.end_test_acc!=None and self.test_loss_list[-1]self.end_test_acc: # check if both the test loss and accuracy meet the end condition - return True + # a method to check whether to end the training according to some conditions + if self.end_loss!=None and len(self.train_loss_list)!=0 and self.train_loss_list[-1]self.end_acc: + return True # return True if the training accuracy is higher than the end accuracy + elif self.end_loss!=None and len(self.train_loss_list)!=0 and self.end_acc!=None and self.train_loss_list[-1]self.end_acc: + return True # return True if both the training loss and accuracy meet the end conditions + elif self.end_test_loss!=None and len(self.test_loss_list)!=0 and self.test_loss_list[-1]self.end_test_acc: + return True # return True if the testing accuracy is higher than the end test accuracy + elif self.end_test_loss!=None and len(self.test_loss_list)!=0 and self.end_test_acc!=None and self.test_loss_list[-1]self.end_test_acc: + return True # return True if both the testing loss and accuracy meet the end conditions - @tf.function + @tf.function(jit_compile=True) def opt_p(self,data,labels,p,lock,g_lock=None): + # a method to perform one optimization step for a given process try: try: - if self.nn.GradientTape!=None: # check if the neural network has its own gradient tape function - tape,output,loss=self.nn.GradientTape(data,labels,p) # use the neural network's gradient tape function to get the tape, output and loss - except Exception: - with tf.GradientTape(persistent=True) as tape: # use the default gradient tape function + with tf.GradientTape(persistent=True) as tape: # create a gradient tape to record the gradients try: try: - output=self.nn.fp(data) # use the neural network's forward propagation function to get the output - loss=self.nn.loss(output,labels) # use the neural network's loss function to get the loss + output=self.nn.fp(data,p) # get the output of the neural network model with data and process index as inputs + loss=self.nn.loss(output,labels,p) # get the loss of the output with labels and process index as inputs except Exception: - output,loss=self.nn.fp(data,labels) # use the neural network's forward propagation function to get the output and loss + output,loss=self.nn.fp(data,labels,p) # get the output and loss of the neural network model with data, labels and process index as inputs except Exception: try: - output=self.nn.fp(data,p) # use the neural network's forward propagation function to get the output with the process number - loss=self.nn.loss(output,labels) # use the neural network's loss function to get the loss + output=self.nn.fp(data) # get the output of the neural network model with data as input + loss=self.nn.loss(output,labels) # get the loss of the output with labels as input except Exception: - output,loss=self.nn.fp(data,labels,p) # use the neural network's forward propagation function to get the output and loss with the process number + output,loss=self.nn.fp(data,labels) # get the output and loss of the neural network model with data and labels as inputs + except Exception: + if hasattr(self.nn,'GradientTape'): + tape,output,loss=self.nn.GradientTape(data,labels,p) # use a custom gradient tape method in the neural network model with data, labels and process index as inputs except Exception as e: - raise e - if self.PO==1: # check if the optimizer order is 1 (lock before gradient calculation) - if self.priority_flag==True and self.priority_p.value!=-1: # check if the priority flag is True and the priority parameter is not -1 - while True: - if p==self.priority_p.value: # check if the process number matches the priority parameter - break - else: - continue - lock[0].acquire() # acquire the first lock - if self.stop_func_(lock[0]): # check if the stop condition is met - return None,0 + raise e # raise any exception that occurs + if self.PO==1: # if using PO=1 strategy (lock before calculating gradients) + if self.priority_flag==True and self.priority_p.value!=-1: # if using priority optimization and there is a priority process index + while True: + if p==self.priority_p.value: # wait until this process is equal to the priority process index + break + else: + continue + lock[0].acquire() # acquire the first lock (for calculating gradients) + if self.stop_func_(lock[0]): # check whether to stop the training by external signal + return None,0 # return None values if stopping try: - try: + if hasattr(self.nn,'gradient'): # if the neural network model has a custom gradient method try: - gradient=self.nn.gradient(tape,loss) # use the neural network's gradient function to get the gradient + gradient=self.nn.gradient(tape,loss) # get the gradients with the tape and loss as inputs except Exception: - gradient=self.nn.gradient(tape,loss,self.param[7]) # use the neural network's gradient function to get the gradient with the parameters - except Exception: - gradient=tape.gradient(loss,self.nn.param) # use the default gradient function to get the gradient - except Exception as e: - raise e - try: - gradient=self.nn.attenuate(gradient,self.nn.opt_counter,p) # use the neural network's attenuate function to modify the gradient + gradient=self.nn.gradient(tape,loss,self.param[7]) # get the gradients with the tape, loss and model parameters as inputs + else: + gradient=tape.gradient(loss,self.nn.param) # get the gradients with the tape and model parameters as inputs except Exception as e: - try: - if self.nn.attenuate!=None: # check if the neural network has an attenuate function - raise e - except Exception: - pass + raise e # raise any exception that occurs + if hasattr(self.nn,'attenuate'): # if the neural network model has a custom attenuate method for modifying the gradients + gradient=self.nn.attenuate(gradient,self.nn.opt_counter,p) # modify the gradients with the optimization counter and process index as inputs try: try: - param=self.nn.opt(gradient) # use the neural network's optimizer function to update the parameters + param=self.nn.opt(gradient) # get the updated model parameters with the gradients as input except Exception: - param=self.nn.opt(gradient,p) # use the neural network's optimizer function to update the parameters with the process number + param=self.nn.opt(gradient,p) # get the updated model parameters with the gradients and process index as inputs except Exception as e: - raise e - lock[0].release() # release the first lock - elif self.PO==2: # check if the optimizer order is 2 (lock after gradient calculation) - g_lock.acquire() # acquire the global lock - if self.stop_func_(g_lock): # check if the stop condition is met - return None,0 + raise e # raise any exception that occurs + lock[0].release() # release the first lock (for calculating gradients) + elif self.PO==2: # if using PO=2 strategy (lock before and after calculating gradients) + g_lock.acquire() # acquire the global lock (for calculating gradients) + if self.stop_func_(g_lock): # check whether to stop the training by external signal + return None,0 # return None values if stopping try: - try: + if hasattr(self.nn,'gradient'): # if the neural network model has a custom gradient method try: - gradient=self.nn.gradient(tape,loss) # use the neural network's gradient function to get the gradient + gradient=self.nn.gradient(tape,loss) # get the gradients with the tape and loss as inputs except Exception: - gradient=self.nn.gradient(tape,loss,self.param[7]) # use the neural network's gradient function to get the gradient with the parameters - except Exception: - gradient=tape.gradient(loss,self.nn.param) # use the default gradient function to get the gradient - except Exception as e: - raise e - g_lock.release() # release the global lock - if self.priority_flag==True and self.priority_p.value!=-1: # check if the priority flag is True and the priority parameter is not -1 - while True: - if p==self.priority_p.value: # check if the process number matches the priority parameter - break - else: - continue - lock[0].acquire() # acquire the first lock - if self.stop_func_(lock[0]): # check if the stop condition is met - return None,0 - try: - gradient=self.nn.attenuate(gradient,self.nn.opt_counter,p) # use the neural network's attenuate function to modify the gradient + gradient=self.nn.gradient(tape,loss,self.param[7]) # get the gradients with the tape, loss and model parameters as inputs + else: + gradient=tape.gradient(loss,self.nn.param) # get the gradients with the tape and model parameters as inputs except Exception as e: - try: - if self.nn.attenuate!=None: # check if the neural network has an attenuate function - raise e - except Exception: - pass + raise e # raise any exception that occurs + g_lock.release() # release the global lock (for calculating gradients) + if self.priority_flag==True and self.priority_p.value!=-1: # if using priority optimization and there is a priority process index + while True: + if p==self.priority_p.value: # wait until this process is equal to the priority process index + break + else: + continue + lock[0].acquire() # acquire the first lock (for updating model parameters) + if self.stop_func_(lock[0]): # check whether to stop the training by external signal + return None,0 # return None values if stopping + if hasattr(self.nn,'attenuate'): # if the neural network model has a custom attenuate method for modifying the gradients + gradient=self.nn.attenuate(gradient,self.nn.opt_counter,p) # modify the gradients with the optimization counter and process index as inputs try: try: - param=self.nn.opt(gradient) # use the neural network's optimizer function to update the parameters + param=self.nn.opt(gradient) # get the updated model parameters with the gradients as input except Exception: - param=self.nn.opt(gradient,p) # use the neural network's optimizer function to update the parameters with the process number + param=self.nn.opt(gradient,p) # get the updated model parameters with the gradients and process index as inputs except Exception as e: - raise e - lock[0].release() # release the first lock - elif self.PO==3: # check if the optimizer order is 3 (no lock) - if self.priority_flag==True and self.priority_p.value!=-1: # check if the priority flag is True and the priority parameter is not -1 - while True: - if p==self.priority_p.value: # check if the process number matches the priority parameter - break - else: - continue - if self.stop_func_(): # check if the stop condition is met - return None,0 + raise e # raise any exception that occurs + lock[0].release() # release the first lock (for updating model parameters) + elif self.PO==3: # if using PO=3 strategy (no lock for calculating gradients) + if self.priority_flag==True and self.priority_p.value!=-1: # if using priority optimization and there is a priority process index + while True: + if p==self.priority_p.value: # wait until this process is equal to the priority process index + break + else: + continue + if self.stop_func_(): # check whether to stop the training by external signal + return None,0 # return None values if stopping try: - try: + if hasattr(self.nn,'gradient'): # if the neural network model has a custom gradient method try: - gradient=self.nn.gradient(tape,loss) # use the neural network's gradient function to get the gradient + gradient=self.nn.gradient(tape,loss) # get the gradients with the tape and loss as inputs except Exception: - gradient=self.nn.gradient(tape,loss,self.param[7]) # use the neural network's gradient function to get the gradient with the parameters - except Exception: - gradient=tape.gradient(loss,self.nn.param) # use the default gradient function to get the gradient - except Exception as e: - raise e - try: - gradient=self.nn.attenuate(gradient,self.nn.opt_counter,p) # use the neural network's attenuate function to modify the gradient + gradient=self.nn.gradient(tape,loss,self.param[7]) # get the gradients with the tape, loss and model parameters as inputs + else: + gradient=tape.gradient(loss,self.nn.param) # get the gradients with the tape and model parameters as inputs except Exception as e: - try: - if self.nn.attenuate!=None: # check if the neural network has an attenuate function - raise e - except Exception: - pass + raise e # raise any exception that occurs + if hasattr(self.nn,'attenuate'): # if the neural network model has a custom attenuate method for modifying the gradients + gradient=self.nn.attenuate(gradient,self.nn.opt_counter,p) # modify the gradients with the optimization counter and process index as inputs try: try: - param=self.nn.opt(gradient) # use the neural network's optimizer function to update the parameters + param=self.nn.opt(gradient) # get the updated model parameters with the gradients as input except Exception: - param=self.nn.opt(gradient,p) # use the neural network's optimizer function to update the parameters with the process number + param=self.nn.opt(gradient,p) # get the updated model parameters with the gradients and process index as inputs except Exception as e: - raise e - return output,loss,param # return output, loss and parameters + raise e # raise any exception that occurs + return output,loss,param # return the output, loss and updated model parameters def opt(self,data,labels,p,lock,g_lock): - if self.PO==2: # check if the optimizer order is 2 (lock after gradient calculation) - if type(g_lock)!=list: # check if g_lock is not a list - pass - elif len(g_lock)==self.process: # check if g_lock has same length as process number - ln=p # assign process number to ln (local number) - g_lock=g_lock[ln] # assign g_lock element at ln index to g_lock - else: - ln=int(np.random.choice(len(g_lock))) # assign a random integer from g_lock length to ln (local number) - g_lock=g_lock[ln] # assign g_lock element at ln index to g_lock - output,loss,param=self.opt_p(data,labels,p,lock,g_lock) # call the opt_p function to get output, loss and parameters - else: - output,loss,param=self.opt_p(data,labels,p,lock) # call the opt_p function to get output, loss and parameters without g_lock - return output,loss,param # return output, loss and parameters + # a method to perform one optimization step for a given process with different PO strategies + if self.PO==2: # if using PO=2 strategy (lock before and after calculating gradients) + if type(g_lock)!=list: + pass # do nothing if g_lock is not a list + elif len(g_lock)==self.process: + ln=p # set ln to be the process index + g_lock=g_lock[ln] # get the global lock for this process + else: + ln=int(np.random.choice(len(g_lock))) # randomly choose a global lock from the list + g_lock=g_lock[ln] # get the global lock for this process + output,loss,param=self.opt_p(data,labels,p,lock,g_lock) # perform one optimization step with data, labels, process index, lock and global lock as inputs + else: + output,loss,param=self.opt_p(data,labels,p,lock) # perform one optimization step with data, labels, process index and lock as inputs + return output,loss,param # return the output, loss and updated model parameters def update_nn_param(self,param=None): - if param==None: # check if param is None - parameter_flat=nest.flatten(self.nn.param) # flatten the neural network's parameters - parameter7_flat=nest.flatten(self.param[7]) # flatten the kernel's parameters - else: - parameter_flat=nest.flatten(self.nn.param) # flatten the neural network's parameters - parameter7_flat=nest.flatten(param) # flatten the given param - for i in range(len(parameter_flat)): # loop over the flattened parameters - if param==None: # check if param is None - state_ops.assign(parameter_flat[i],parameter7_flat[i]) # assign the kernel's parameters to the neural network's parameters - else: - state_ops.assign(parameter_flat[i],parameter7_flat[i]) # assign the given param to the neural network's parameters - self.nn.param=nest.pack_sequence_as(self.nn.param,parameter_flat) # pack the flattened parameters back to the neural network's parameters - self.param[7]=nest.pack_sequence_as(self.param[7],parameter7_flat) # pack the flattened parameters back to the kernel's parameters + # a method to update the model parameters of the neural network model with the shared dictionary values or a given parameter value + if param==None: + parameter_flat=nest.flatten(self.nn.param) # flatten the model parameters of the neural network model into a list + parameter7_flat=nest.flatten(self.param[7]) # flatten the 7th value of the shared dictionary into a list + else: + parameter_flat=nest.flatten(self.nn.param) # flatten the model parameters of the neural network model into a list + parameter7_flat=nest.flatten(param) # flatten the given parameter value into a list + for i in range(len(parameter_flat)): + if param==None: + state_ops.assign(parameter_flat[i],parameter7_flat[i]) # assign each element of parameter7_flat to each element of parameter_flat + else: + state_ops.assign(parameter_flat[i],parameter7_flat[i]) # assign each element of parameter7_flat to each element of parameter_flat + self.nn.param=nest.pack_sequence_as(self.nn.param,parameter_flat) # pack the parameter_flat list back into the original structure of self.nn.param + self.param[7]=nest.pack_sequence_as(self.param[7],parameter7_flat) # pack the parameter7_flat list back into the original structure of self.param[7] return def train7(self,train_ds,p,test_batch,lock,g_lock): - while True: # loop until break - for data_batch,labels_batch in train_ds: # loop over the train dataset batches - try: - data_batch,labels_batch=self.nn.data_func(data_batch,labels_batch) # use the neural network's data function to process data and labels - except Exception as e: - try: - if self.nn.data_func!=None: # check if the neural network has a data function - raise e - except Exception: - pass - if self.priority_flag==True: # check if the priority flag is True - self.priority_p.value=np.argmax(self.opt_counter) # assign the index of the maximum value in opt_counter to priority parameter - if self.max_opt!=None and self.opt_counter[self.priority_p.value]>=self.max_opt: # check if max_opt is not None and opt_counter at priority parameter index is greater than or equal to max_opt - self.priority_p.value=int(self.priority_p.value) # convert priority parameter to integer type - elif self.max_opt==None: # check if max_opt is None - self.priority_p.value=int(self.priority_p.value) # convert priority parameter to integer type - else: - self.priority_p.value=-1 # assign -1 to priority parameter - if self.priority_flag==True: # check if the priority flag is True - self.opt_counter[p]=0 # assign zero to opt_counter at process number - try: - opt_counter=self.nn.opt_counter[0] # get the neural network's optimization counter - opt_counter.scatter_update(tf.IndexedSlices(0,p)) # update opt_counter with zero at process number - self.nn.opt_counter[0]=opt_counter # assign opt_counter back to neural network's optimization counter - except Exception as e: - try: - if self.nn.attenuate!=None: # check if the neural network has an attenuate function - raise e - except Exception: - pass - output,batch_loss,param=self.opt(data_batch,labels_batch,p,lock,g_lock) # call the opt function to get output, batch loss and parameters - self.param[7]=param # assign param to kernel's parameters - if self.priority_flag==True: # check if the priority flag is True - opt_counter=np.frombuffer(self.opt_counter.get_obj(),dtype='i') # get a numpy array from opt_counter shared array object - opt_counter+=1 # increment opt_counter by one for each element - try: - opt_counter=self.nn.opt_counter[0] # get the neural network's optimization counter - opt_counter.assign(opt_counter+1) # increment opt_counter by one - self.nn.opt_counter[0]=opt_counter # assign opt_counter back to neural network's optimization counter - except Exception as e: - try: - if self.nn.attenuate!=None: # check if the neural network has an attenuate function - raise e - except Exception: - pass - try: - bc=self.nn.bc[0] # get the neural network's batch counter - bc.assign_add(1) # increment bc by one - self.nn.bc[0]=bc # assign bc back to neural network's batch counter - except Exception: - pass - try: - batch_acc=self.nn.accuracy(output,labels_batch) # use the neural network's accuracy function to get the batch accuracy - except Exception as e: - try: - if self.nn.accuracy!=None: # check if the neural network has an accuracy function - raise e - except Exception: - pass + # a method to perform one epoch of training for a given process with a given training dataset + while True: + for data_batch,labels_batch in train_ds: # iterate over each batch of data and labels in the training dataset + if hasattr(self.nn,'data_func'): + data_batch,labels_batch=self.nn.data_func(data_batch,labels_batch) # apply a custom data function in the neural network model to the data and labels batch + if self.priority_flag==True: + self.priority_p.value=np.argmax(self.opt_counter) # set the priority process index to be the one with the maximum optimization counter + if self.max_opt!=None and self.opt_counter[self.priority_p.value]>=self.max_opt: + self.priority_p.value=int(self.priority_p.value) # keep the priority process index as an integer if it reaches the maximum optimization steps + elif self.max_opt==None: + self.priority_p.value=int(self.priority_p.value) # keep the priority process index as an integer if there is no maximum optimization steps + else: + self.priority_p.value=-1 # set the priority process index to -1 if there is no process with maximum optimization steps + if self.priority_flag==True: + self.opt_counter[p]=0 # reset the optimization counter for this process if using priority optimization + if hasattr(self.nn,'attenuate'): + opt_counter=self.nn.opt_counter[0] # get the optimization counter in the neural network model + opt_counter.scatter_update(tf.IndexedSlices(0,p)) # update the optimization counter for this process to 0 + self.nn.opt_counter[0]=opt_counter # set the optimization counter in the neural network model + output,batch_loss,param=self.opt(data_batch,labels_batch,p,lock,g_lock) # perform one optimization step with data and labels batch, process index, lock and global lock as inputs + self.param[7]=param # set the 7th value of the shared dictionary to be the updated model parameters + if self.priority_flag==True: + opt_counter=np.frombuffer(self.opt_counter.get_obj(),dtype='i') # get the optimization counter from the shared array + opt_counter+=1 # increment the optimization counter by 1 for each process + if hasattr(self.nn,'attenuate'): + opt_counter=self.nn.opt_counter[0] # get the optimization counter in the neural network model + opt_counter.assign(opt_counter+1) # increment the optimization counter by 1 for this process + self.nn.opt_counter[0]=opt_counter # set the optimization counter in the neural network model + if hasattr(self.nn,'bc'): + bc=self.nn.bc[0] # get the batch counter in the neural network model + bc.assign_add(1) # increment the batch counter by 1 for this process + self.nn.bc[0]=bc # set the batch counter in the neural network model try: - if self.nn.accuracy!=None: # check if the neural network has an accuracy function - self.total_loss[p]+=batch_loss # accumulate batch loss to total loss at process number - self.total_acc[p]+=batch_acc # accumulate batch accuracy to total accuracy at process number - except Exception: - self.total_loss[p]+=batch_loss # accumulate batch loss to total loss at process number - self.batch_counter[p]+=1 # increment batch counter at process number - if self.PO==1 or self.PO==2: # check if the optimizer order is 1 or 2 (lock before or after gradient calculation) - lock[1].acquire() # acquire the second lock - elif lock!=None: # check if lock is not None - lock.acquire() # acquire the lock - batches=np.sum(self.batch_counter) # sum up the batch counter for all processes - if batches>=self.batches: # check if the number of batches reaches the total number of batches - batch_counter=np.frombuffer(self.batch_counter.get_obj(),dtype='i') # get a numpy array from batch counter shared array object - batch_counter*=0 # reset batch counter to zero for each element - loss=np.sum(self.total_loss)/batches # calculate the average loss for all batches - try: - if self.nn.accuracy!=None: # check if the neural network has an accuracy function - train_acc=np.sum(self.total_acc)/batches # calculate the average accuracy for all batches - except Exception: - pass - self.total_epoch.value+=1 # increment total epoch by one - self.train_loss.value=loss # assign loss to train loss - self.train_loss_list.append(loss) # append loss to train loss list - try: - if self.nn.accuracy!=None: # check if the neural network has an accuracy function - self.train_acc.value=train_acc # assign train_acc to train accuracy - self.train_acc_list.append(train_acc) # append train_acc to train accuracy list - except Exception: - pass - if self.test_flag==True: # check if the test flag is True - self.test_loss.value,self.test_acc.value=self.test(self.test_data,self.test_labels,test_batch,p) # call the test function to get test loss and accuracy - self.test_loss_list.append(self.test_loss.value) # append test loss to test loss list - try: - if self.nn.accuracy!=None: # check if the neural network has an accuracy function - self.test_acc_list.append(self.test_acc.value) # append test accuracy to test accuracy list - except Exception: - pass - self.print_save() # call the print_save function to print and save the results - self.epoch_counter.value+=1 # increment epoch counter by one - try: - ec=self.nn.ec[0] # get the neural network's epoch counter - ec.assign_add(1) # increment ec by one - self.nn.ec[0]=ec # assign ec back to neural network's epoch counter - except Exception: - pass - total_loss=np.frombuffer(self.total_loss.get_obj(),dtype='f') # get a numpy array from total loss shared array object - total_loss*=0 # reset total loss to zero for each element - try: - total_acc=np.frombuffer(self.total_acc.get_obj(),dtype='f') # get a numpy array from total accuracy shared array object - total_acc*=0 # reset total accuracy to zero for each element - except Exception as e: + if hasattr(self.nn,'accuracy'): try: - if self.nn.accuracy!=None: # check if the neural network has an accuracy function - raise e + batch_acc=self.nn.accuracy(output,labels_batch,p) # get the accuracy of the output with labels and process index as inputs except Exception: - pass - if self.PO==1 or self.PO==2: # check if the optimizer order is 1 or 2 (lock before or after gradient calculation) - lock[1].release() # release the second lock - elif lock!=None: # check if lock is not None - lock.release() # release the lock - if self.epoch_counter.value>=self.epoch: # check if the epoch counter reaches the epoch number - self.param[7]=param # assign param to kernel's parameters - return + batch_acc=self.nn.accuracy(output,labels_batch) # get the accuracy of the output with labels as inputs + except Exception as e: + raise e # raise any exception that occurs + if hasattr(self.nn,'accuracy'): + self.total_loss[p]+=batch_loss # accumulate the total loss for this process + self.total_acc[p]+=batch_acc # accumulate the total accuracy for this process + else: + self.total_loss[p]+=batch_loss # accumulate the total loss for this process + self.batch_counter[p]+=1 # increment the batch counter for this process + if self.PO==1 or self.PO==2: + lock[1].acquire() # acquire the second lock (for printing and saving) + elif lock!=None: + lock.acquire() # acquire a single lock (for printing and saving) + batches=np.sum(self.batch_counter) # get the total number of batches for all processes + if batches>=self.batches: # check whether all batches are finished for this epoch + batch_counter=np.frombuffer(self.batch_counter.get_obj(),dtype='i') # get the batch counter from the shared array + batch_counter*=0 # reset the batch counter to 0 for each process + loss=np.sum(self.total_loss)/batches # calculate the average training loss for this epoch + if hasattr(self.nn,'accuracy'): + train_acc=np.sum(self.total_acc)/batches # calculate the average training accuracy for this epoch + self.total_epoch.value+=1 # increment the total epoch by 1 + self.train_loss.value=loss # set the training loss value to be the average loss + self.train_loss_list.append(loss) # append the average loss to the training loss list + if hasattr(self.nn,'accuracy'): + self.train_acc.value=train_acc # set the training accuracy value to be the average accuracy + self.train_acc_list.append(train_acc) # append the average accuracy to the training accuracy list + if self.test_flag==True: # if using testing data or dataset + if hasattr(self.nn,'accuracy'): + self.test_loss.value,self.test_acc.value=self.test(self.test_data,self.test_labels,test_batch) # get the testing loss and accuracy values with testing data, labels and batch size as inputs + self.test_loss_list.append(self.test_loss.value) # append the testing loss value to the testing loss list + self.test_acc_list.append(self.test_acc.value) # append the testing accuracy value to the testing accuracy list + else: + self.test_loss.value=self.test(self.test_data,self.test_labels,test_batch) # get the testing loss value with testing data, labels and batch size as inputs + self.test_loss_list.append(self.test_loss.value) # append the testing loss value to the testing loss list + self.print_save() # print and save the results for this epoch + self.epoch_counter.value+=1 # increment the epoch counter by 1 + if hasattr(self.nn,'ec'): + ec=self.nn.ec[0] # get the epoch counter in the neural network model + ec.assign_add(1) # increment the epoch counter by 1 for this process + self.nn.ec[0]=ec # set the epoch counter in the neural network model + total_loss=np.frombuffer(self.total_loss.get_obj(),dtype='f') # get the total loss from the shared array + total_loss*=0 # reset the total loss to 0 for each process + if hasattr(self.nn,'accuracy'): + total_acc=np.frombuffer(self.total_acc.get_obj(),dtype='f') # get the total accuracy from the shared array + total_acc*=0 # reset the total accuracy to 0 for each process + if self.PO==1 or self.PO==2: + lock[1].release() # release the second lock (for printing and saving) + elif lock!=None: + lock.release() # release a single lock (for printing and saving) + if self.epoch_counter.value>=self.epoch: + self.param[7]=param # set the 7th value of the shared dictionary to be the updated model parameters + return # return from this method def train(self,p,lock=None,g_lock=None,test_batch=None): - if self.epoch!=None: # check if epoch is not None - if self.train_dataset!=None: # check if train dataset is not None - train_ds=self.train_dataset # assign train dataset to train_ds - else: - if self.data_segment_flag==True: # check if data segment flag is True - train_ds=tf.data.Dataset.from_tensor_slices((self.train_data[p],self.train_labels[p])).batch(self.batch) # create a dataset from tensor slices of segmented data and labels and batch them - elif self.buffer_size!=None: # check if buffer size is not None - train_ds=tf.data.Dataset.from_tensor_slices((self.train_data,self.train_labels)).shuffle(self.buffer_size).batch(self.batch) # create a dataset from tensor slices of data and labels and shuffle and batch them with buffer size - else: - train_ds=tf.data.Dataset.from_tensor_slices((self.train_data,self.train_labels)).batch(self.batch) # create a dataset from tensor slices of data and labels and batch them - self.train7(train_ds,p,test_batch,lock,g_lock) # call the train7 function with train_ds, process number, test batch, lock and g_lock + # a method to perform one epoch of training for a given process with different PO strategies and data sources + if self.train_dataset is not None and type(self.train_dataset)==list: + train_ds=self.train_dataset[p] # get the training dataset for this process if it is a list of datasets + elif self.train_dataset is not None: + train_ds=self.train_dataset # get the training dataset if it is a single dataset + else: + if self.data_segment_flag==True: + train_ds=tf.data.Dataset.from_tensor_slices((self.train_data[p],self.train_labels[p])).batch(self.batch).prefetch(self.prefetch_batch_size) # create a tf.data.Dataset object from segmented data and labels for this process with batch size and prefetch size as inputs + elif self.buffer_size!=None: + train_ds=tf.data.Dataset.from_tensor_slices((self.train_data,self.train_labels)).shuffle(self.buffer_size).batch(self.batch).prefetch(self.prefetch_batch_size) # create a tf.data.Dataset object from shuffled data and labels with buffer size, batch size and prefetch size as inputs + else: + train_ds=tf.data.Dataset.from_tensor_slices((self.train_data,self.train_labels)).batch(self.batch).prefetch(self.prefetch_batch_size) # create a tf.data.Dataset object from data and labels with batch size and prefetch size as inputs + self.train7(train_ds,p,test_batch,lock,g_lock) # perform one epoch of training for this process with the training dataset, process index, test batch size, lock and global lock as inputs return - def test(self,test_data=None,test_labels=None,batch=None,p=None): - if test_data is not None and type(self.nn.param[0])!=list: - test_data=test_data.astype(self.nn.param[0].dtype.name) # convert the test data type to match the model parameter type - test_labels=test_labels.astype(self.nn.param[0].dtype.name) # convert the test labels type to match the model parameter type - elif test_data is not None: - test_data=test_data.astype(self.nn.param[0][0].dtype.name) # convert the test data type to match the model parameter type - test_labels=test_labels.astype(self.nn.param[0][0].dtype.name) # convert the test labels type to match the model parameter type - if batch!=None: # check if batch is not None - total_loss=0 # initialize total loss to zero - total_acc=0 # initialize total accuracy to zero - if self.test_dataset!=None: # check if test dataset is not None - batches=0 - for data_batch,labels_batch in self.test_dataset: # loop over the test dataset batches - batches+=1 - try: - try: - output=self.nn.fp(data_batch) # use the neural network's forward propagation function to get the output - except Exception: - output=self.nn.fp(data_batch,p) # use the neural network's forward propagation function to get the output with the process number - except Exception as e: - raise e - batch_loss=self.nn.loss(output,labels_batch) # use the neural network's loss function to get the batch loss - total_loss+=batch_loss # accumulate batch loss to total loss - try: - batch_acc=self.nn.accuracy(output,labels_batch) # use the neural network's accuracy function to get the batch accuracy - total_acc+=batch_acc # accumulate batch accuracy to total accuracy - except Exception as e: - try: - if self.nn.accuracy!=None: # check if the neural network has an accuracy function - raise e - except Exception: - pass - test_loss=total_loss.numpy()/batches # calculate the average test loss for all batches - try: - if self.nn.accuracy!=None: # check if the neural network has an accuracy function - test_acc=total_acc.numpy()/batches # calculate the average test accuracy for all batches - except Exception: - pass - else: # if test dataset is None - batches=int((test_data.shape[0]-test_data.shape[0]%batch)/batch) # calculate the number of batches - shape0=test_data.shape[0] # get the number of samples in the test data array - for j in range(batches): # loop over the batches - index1=j*batch # get the start index of the batch - index2=(j+1)*batch # get the end index of the batch - data_batch=test_data[index1:index2] # get the data batch from test data array - labels_batch=test_labels[index1:index2] # get the labels batch from test labels array - try: - try: - output=self.nn.fp(data_batch) # use the neural network's forward propagation function to get the output - except Exception: - output=self.nn.fp(data_batch,p) # use the neural network's forward propagation function to get the output with the process number - except Exception as e: - raise e - batch_loss=self.nn.loss(output,labels_batch) # use the neural network's loss function to get the batch loss - total_loss+=batch_loss # accumulate batch loss to total loss - try: - batch_acc=self.nn.accuracy(output,labels_batch) # use the neural network's accuracy function to get the batch accuracy - total_acc+=batch_acc # accumulate batch accuracy to total accuracy - except Exception as e: - try: - if self.nn.accuracy!=None: # check if the neural network has an accuracy function - raise e - except Exception: - pass - if shape0%batch!=0: # check if there are remaining samples that are not in a batch - batches+=1 # increment batches by one - index1=batches*batch # get the start index of the remaining samples - index2=batch-(shape0-batches*batch) # get the number of samples that need to be filled with extra samples - data_batch=tf.concat([test_data[index1:],test_data[:index2]],0) # concatenate the remaining samples and extra samples to form a data batch - labels_batch=tf.concat([test_labels[index1:],test_labels[:index2]],0) # concatenate the remaining labels and extra labels to form a labels batch - try: - try: - output=self.nn.fp(data_batch) # use the neural network's forward propagation function to get the output - except Exception: - output=self.nn.fp(data_batch,p) # use the neural network's forward propagation function to get the output with the process number - except Exception as e: - raise e - batch_loss=self.nn.loss(output,labels_batch) # use the neural network's loss function to get the batch loss - total_loss+=batch_loss # accumulate batch loss to total loss - try: - batch_acc=self.nn.accuracy(output,labels_batch) # use the neural network's accuracy function to get the batch accuracy - total_acc+=batch_acc # accumulate batch accuracy to total accuracy - except Exception as e: - try: - if self.nn.accuracy!=None: # check if the neural network has an accuracy function - raise e - except Exception: - pass - test_loss=total_loss.numpy()/batches # calculate the average test loss for all batches + def train_online(self,p,lock=None,g_lock=None): + # a method to perform online training for a given process with different PO strategies + if hasattr(self.nn,'counter'): + self.nn.counter.append(0) # append a 0 to the counter list in the neural network model for this process + while True: + if hasattr(self.nn,'save'): + self.nn.save(self.save,p) # save the model parameters with the save path and process index as inputs if the neural network model has a save method + if hasattr(self.nn,'stop_flag'): + if self.nn.stop_flag==True: # check whether to stop the training by the stop flag in the neural network model + return # return from this method + if hasattr(self.nn,'stop_func'): + if self.nn.stop_func(p): # check whether to stop the training by the stop function in the neural network model with process index as input + return # return from this method + if hasattr(self.nn,'suspend_func'): + self.nn.suspend_func(p) # suspend the training by the suspend function in the neural network model with process index as input try: - if self.nn.accuracy!=None: # check if the neural network has an accuracy function - test_acc=total_acc.numpy()/batches # calculate the average test accuracy for all batches - except Exception: - pass - else: # if batch is None + data=self.nn.online(p) # get the online data from the online method in the neural network model with process index as input + except Exception as e: + self.nn.exception_list[p]=e # append any exception that occurs to the exception list in the neural network model for this process + if data=='stop': + return # return from this method if the online data is 'stop' + elif data=='suspend': + self.nn.suspend_func(p) # suspend the training by the suspend function in the neural network model with process index as input if the online data is 'suspend' try: - try: - output=self.nn.fp(test_data) # use the neural network's forward propagation function to get the output - except Exception: - output=self.nn.fp(test_data,p) # use the neural network's forward propagation function to get the output with the process number + if self.PO==2: # if using PO=2 strategy (lock before and after calculating gradients) + if type(g_lock)!=list: + pass # do nothing if g_lock is not a list + elif len(g_lock)==self.process: + ln=p # set ln to be the process index + g_lock=g_lock[ln] # get the global lock for this process + else: + ln=int(np.random.choice(len(g_lock))) # randomly choose a global lock from the list + g_lock=g_lock[ln] # get the global lock for this process + output,loss,param=self.opt(data[0],data[1],p,lock,g_lock) # perform one optimization step with data and labels, process index, lock and global lock as inputs + self.param[7]=param # set the 7th value of the shared dictionary to be the updated model parameters except Exception as e: - raise e - test_loss=self.nn.loss(output,test_labels) # use the neural network's loss function to get the test loss - test_loss=test_loss.numpy() # convert test loss to numpy array + if self.PO==1: + if lock[0].acquire(False): + lock[0].release() # release the first lock if it is acquired by this process + elif self.PO==2: + if g_lock.acquire(False): + g_lock.release() # release the global lock if it is acquired by this process + if lock[0].acquire(False): + lock[0].release() # release the first lock if it is acquired by this process + self.nn.exception_list[p]=e # append any exception that occurs to the exception list in the neural network model for this process + loss=loss.numpy() # convert the loss value to a numpy array + if len(self.nn.train_loss_list)==self.nn.max_length: + del self.nn.train_loss_list[0] # delete the first element of the training loss list in the neural network model if it reaches the maximum length + self.nn.train_loss_list.append(loss) # append the loss value to the training loss list in the neural network model try: - test_acc=self.nn.accuracy(output,test_labels) # use the neural network's accuracy function to get the test accuracy - test_acc=test_acc.numpy() # convert test accuracy to numpy array + if hasattr(self.nn,'accuracy'): + try: + acc=self.nn.accuracy(output,data[1]) # get the accuracy of the output with labels as inputs + except Exception: + self.exception_list[p]=True # set the exception flag to True for this process if there is an exception + if len(self.nn.train_acc_list)==self.nn.max_length: + del self.nn.train_acc_list[0] # delete the first element of the training accuracy list in the neural network model if it reaches the maximum length + self.nn.train_acc_list.append(acc) # append the accuracy value to the training accuracy list in the neural network model except Exception as e: - try: - if self.nn.accuracy!=None: # check if the neural network has an accuracy function - raise e - except Exception: - pass - try: - if self.nn.accuracy!=None: # check if the neural network has an accuracy function - return test_loss,test_acc # return test loss and accuracy - except Exception: - return test_loss,None # return test loss and None - - - def stop_func(self): - if self.end(): # check if any of the end conditions is met - self.save(self.total_epoch.value,True) # save the model with total epoch and True flag - self.save_flag.value=True # set save flag to True - self.stop_flag.value=True # set stop flag to True - return True - return False + self.nn.exception_list[p]=e # append any exception that occurs to the exception list in the neural network model for this process + try: + if hasattr(self.nn,'counter'): + count=self.nn.counter[p] # get the counter value for this process in the neural network model + count+=1 # increment the counter value by 1 + self.nn.counter[p]=count # set the counter value for this process in the neural network model + except IndexError: + self.nn.counter.append(0) # append a 0 to the counter list in the neural network model for this process if there is an index error + count=self.nn.counter[p] # get the counter value for this process in the neural network model + count+=1 # increment the counter value by 1 + self.nn.counter[p]=count # set the counter value for this process in the neural network model + return # return from this method - def stop_func_(self,lock=None): - if self.stop==True: # check if stop is True - if self.stop_flag.value==True or self.stop_func(): # check if the stop flag is True or the stop function returns True - if self.PO!=3: # check if the optimizer order is not 3 (no lock) - lock.release() # release the lock - return True - return False + @tf.function(jit_compile=True) + def test_(self,data,labels): + # a method to perform one testing step with data and labels as inputs + try: + try: + output=self.nn.fp(data) # get the output of the neural network model with data as input + loss=self.nn.loss(output,labels) # get the loss of the output with labels as input + except Exception: + output,loss=self.nn.fp(data,labels) # get the output and loss of the neural network model with data and labels as inputs + except Exception as e: + raise e # raise any exception that occurs + try: + acc=self.nn.accuracy(output,labels) # get the accuracy of the output with labels as input + except Exception as e: + if hasattr(self.nn,'accuracy'): + raise e # raise any exception that occurs if using accuracy metric + else: + acc=None # set acc to None if not using accuracy metric + return loss,acc # return the loss and accuracy values - def visualize_train(self): - print() # print a blank line - plt.figure(1) # create a new figure - plt.plot(np.arange(self.total_epoch.value),self.train_loss_list) # plot the train loss list against the total epoch - plt.title('train loss') # set the title of the figure - plt.xlabel('epoch') # set the x-axis label of the figure - plt.ylabel('loss') # set the y-axis label of the figure - print('train loss:{0:.6f}'.format(self.train_loss.value)) # print the train loss value with six decimal places - try: - if self.nn.accuracy!=None: # check if the neural network has an accuracy function - plt.figure(2) # create a new figure - plt.plot(np.arange(self.total_epoch.value),self.train_acc_list) # plot the train accuracy list against the total epoch - plt.title('train acc') # set the title of the figure - plt.xlabel('epoch') # set the x-axis label of the figure - plt.ylabel('acc') # set the y-axis label of the figure - if self.acc_flag=='%': # check if the accuracy format is percentage - print('train acc:{0:.1f}'.format(self.train_acc.value*100)) # print the train accuracy value with one decimal place and percentage sign - else: - print('train acc:{0:.6f}'.format(self.train_acc.value)) # print the train accuracy value with six decimal places - except Exception: - pass - return + def test(self,test_data=None,test_labels=None,batch=None): + # a method to perform testing with different data sources and batch sizes + if test_data is not None and type(self.nn.param[0])!=list: + test_data=test_data.astype(self.nn.param[0].dtype.name) # convert the testing data to the same dtype as the model parameters + test_labels=test_labels.astype(self.nn.param[0].dtype.name) # convert the testing labels to the same dtype as the model parameters + elif test_data is not None: + test_data=test_data.astype(self.nn.param[0][0].dtype.name) # convert the testing data to the same dtype as the model parameters + test_labels=test_labels.astype(self.nn.param[0][0].dtype.name) # convert the testing labels to the same dtype as the model parameters + if self.process_t!=None: # if using multiple processes for testing + parallel_test_=parallel_test(self.nn,self.test_data,self.test_labels,self.process_t,batch,self.prefetch_batch_size_t,self.test_dataset) # create a parallel_test object with the neural network model, testing data, labels, number of processes, batch size, prefetch size and dataset as inputs + if type(self.test_data)!=list: + parallel_test_.segment_data() # segment the data for each process if it is not a list of data + for p in range(self.process_t): + Process(target=parallel_test_.test,args=(p,)).start() # start a subprocess to perform testing for each process + try: + if hasattr(self.nn,'accuracy'): + test_loss,test_acc=parallel_test_.loss_acc() # get the testing loss and accuracy values from the parallel_test object + except Exception as e: + if hasattr(self.nn,'accuracy'): + raise e # raise any exception that occurs if using accuracy metric + else: + test_loss=parallel_test_.loss_acc() # get the testing loss value from the parallel_test object + elif batch!=None: # if using a batch size for testing + total_loss=0 # initialize a total loss value + total_acc=0 # initialize a total accuracy value + if self.test_dataset!=None: # if using a testing dataset + batches=0 # initialize a batches counter + for data_batch,labels_batch in self.test_dataset: # iterate over each batch of data and labels in the testing dataset + batches+=1 # increment the batches counter by 1 + batch_loss,batch_acc=self.test_(data_batch,labels_batch) # perform one testing step with data and labels batch as inputs + total_loss+=batch_loss # accumulate the total loss value + if hasattr(self.nn,'accuracy'): + total_acc+=batch_acc # accumulate the total accuracy value if using accuracy metric + test_loss=total_loss.numpy()/batches # calculate the average testing loss value + if hasattr(self.nn,'accuracy'): + test_acc=total_acc.numpy()/batches # calculate the average testing accuracy value if using accuracy metric + else: + shape0=test_data.shape[0] # get the number of samples in testing data + batches=int((shape0-shape0%batch)/batch) # calculate the number of batches for testing data + for j in range(batches): + index1=j*batch # get the start index of each batch + index2=(j+1)*batch # get the end index of each batch + data_batch=test_data[index1:index2] # get a slice of testing data for each batch + labels_batch=test_labels[index1:index2] # get a slice of testing labels for each batch + batch_loss,batch_acc=self.test_(data_batch,labels_batch) # perform one testing step with data and labels batch as inputs + total_loss+=batch_loss # accumulate the total loss value + if hasattr(self.nn,'accuracy'): + total_acc+=batch_acc # accumulate the total accuracy value if using accuracy metric + if shape0%batch!=0: # if there are some remaining samples in testing data + batches+=1 # add one more batch for them + index1=batches*batch # get the start index of the last batch + index2=batch-(shape0-batches*batch) # get the end index of the last batch + data_batch=tf.concat([test_data[index1:],test_data[:index2]],0) # concatenate the remaining samples and some samples from the beginning of testing data to form the last batch + labels_batch=tf.concat([test_labels[index1:],test_labels[:index2]],0) # concatenate the corresponding labels to form the last batch + batch_loss,batch_acc=self.test_(data_batch,labels_batch) # perform one testing step with data and labels batch as inputs + total_loss+=batch_loss # accumulate the total loss value + if hasattr(self.nn,'accuracy'): + total_acc+=batch_acc # accumulate the total accuracy value if using accuracy metric + test_loss=total_loss.numpy()/batches # calculate the average testing loss value + if hasattr(self.nn,'accuracy'): + test_acc=total_acc.numpy()/batches # calculate the average testing accuracy value if using accuracy metric + else: + batch_loss,batch_acc=self.test_(test_data,test_labels) # perform one testing step with testing data and labels as inputs + test_loss=test_loss.numpy() # convert the testing loss value to a numpy array + if hasattr(self.nn,'accuracy'): + test_acc=test_acc.numpy() # convert the testing accuracy value to a numpy array if using accuracy metric + if hasattr(self.nn,'accuracy'): + return test_loss,test_acc # return the testing loss and accuracy values if using accuracy metric + else: + return test_loss # return the testing loss value if not using accuracy metric From ad35e26956a1618ae19a6808aa3d51ea08204b82 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Sat, 22 Jul 2023 23:52:18 +0800 Subject: [PATCH 075/337] Update kernel.py --- .../RL/kernel/nspn/kernel.py | 2002 ++++++++--------- 1 file changed, 1001 insertions(+), 1001 deletions(-) diff --git a/Note 7.0 documentation/RL/kernel/nspn/kernel.py b/Note 7.0 documentation/RL/kernel/nspn/kernel.py index f7ccf658..a6ee7dc5 100644 --- a/Note 7.0 documentation/RL/kernel/nspn/kernel.py +++ b/Note 7.0 documentation/RL/kernel/nspn/kernel.py @@ -63,281 +63,191 @@ def __init__(self,nn=None,save_episode=False): # initialize the total_time attribute as 0 self.total_time=0 - # define a method named action_vec - def action_vec(self): - # if the epsilon attribute is not None, assign a numpy array of ones with the length of action_count and multiply it by epsilon divided by the length of action_one to the action_one attribute - if self.epsilon!=None: - self.action_one=np.ones(self.action_count,dtype=np.int8) - # return nothing - return - - # define a method named init - def init(self): - # try to execute the following statements - try: - # if the nn attribute has a pr attribute, assign a numpy array of 0 to its TD attribute - if hasattr(self.nn,'pr'): - self.nn.pr.TD=np.array(0) - # if an exception occurs, raise it - except Exception as e: + # define a method named action_vec + def action_vec(self): + # if the epsilon attribute is not None, assign a numpy array of ones with the length of action_count and multiply it by epsilon divided by the length of action_one to the action_one attribute + if self.epsilon!=None: + self.action_one=np.ones(self.action_count,dtype=np.int8) + # return nothing + return + + # define a method named init + def init(self): + # try to execute the following statements + try: + # if the nn attribute has a pr attribute, assign a numpy array of 0 to its TD attribute + if hasattr(self.nn,'pr'): + self.nn.pr.TD=np.array(0) + # if an exception occurs, raise it + except Exception as e: + raise e + # assign False to the suspend attribute + self.suspend=False + # assign None to the save_epi attribute + self.save_epi=None + # assign an empty list to the episode_set attribute + self.episode_set=[] + # assign None to the state_pool, action_pool, next_state_pool, reward_pool, done_pool attributes + self.state_pool=None + self.action_pool=None + self.next_state_pool=None + self.reward_pool=None + self.done_pool=None + # assign an empty list to the reward_list attribute + self.reward_list=[] + # assign 0 to the loss attribute + self.loss=0 + # assign an empty list to the loss_list attribute + self.loss_list=[] + # assign 0 to the sc attribute + self.sc=0 + # assign 0 to the total_episode attribute + self.total_episode=0 + # assign 0 to the time attribute + self.time=0 + # assign 0 to the total_time attribute + self.total_time=0 + + # define a method named set_up + def set_up(self,epsilon=None,episode_step=None,pool_size=None,batch=None,update_step=None,trial_count=None,criterion=None): + # if epsilon is not None, assign it to the epsilon attribute + if epsilon!=None: + self.epsilon=epsilon + # if episode_step is not None, assign it to the episode_step attribute + if episode_step!=None: + self.episode_step=episode_step + # if pool_size is not None, assign it to the pool_size attribute + if pool_size!=None: + self.pool_size=pool_size + # if batch is not None, assign it to the batch attribute + if batch!=None: + self.batch=batch + # if update_step is not None, assign it to the update_step attribute + if update_step!=None: + self.update_step=update_step + # if trial_count is not None, assign it to the trial_count attribute + if trial_count!=None: + self.trial_count=trial_count + # if criterion is not None, assign it to the criterion attribute + if criterion!=None: + self.criterion=criterion + # call the action_vec method + self.action_vec() + # return nothing + return + + # define a method named epsilon_greedy_policy + def epsilon_greedy_policy(self,s): + # assign a numpy array of ones with the length of action_count and multiply it by epsilon divided by the length of action_one to the action_prob variable + action_prob=self.action_one*self.epsilon/len(self.action_one) + # try to execute the following statements + try: + # assign the index of the maximum value of the output of the nn attribute's fp method with s as input to the best_a variable + best_a=np.argmax(self.nn.nn.fp(s)) + # add 1 minus epsilon to the best_a element of action_prob + action_prob[best_a]+=1-self.epsilon + # if an exception occurs, execute the following statements + except Exception as e: + # if the platform attribute has a DType attribute, raise the exception + if hasattr(self.platform,'DType'): raise e - # assign False to the suspend attribute - self.suspend=False - # assign None to the save_epi attribute - self.save_epi=None - # assign an empty list to the episode_set attribute - self.episode_set=[] - # assign None to the state_pool, action_pool, next_state_pool, reward_pool, done_pool attributes - self.state_pool=None - self.action_pool=None - self.next_state_pool=None - self.reward_pool=None - self.done_pool=None - # assign an empty list to the reward_list attribute - self.reward_list=[] - # assign 0 to the loss attribute - self.loss=0 - # assign an empty list to the loss_list attribute - self.loss_list=[] - # assign 0 to the sc attribute - self.sc=0 - # assign 0 to the total_episode attribute - self.total_episode=0 - # assign 0 to the time attribute - self.time=0 - # assign 0 to the total_time attribute - self.total_time=0 - - # define a method named set_up - def set_up(self,epsilon=None,episode_step=None,pool_size=None,batch=None,update_step=None,trial_count=None,criterion=None): - # if epsilon is not None, assign it to the epsilon attribute - if epsilon!=None: - self.epsilon=epsilon - # if episode_step is not None, assign it to the episode_step attribute - if episode_step!=None: - self.episode_step=episode_step - # if pool_size is not None, assign it to the pool_size attribute - if pool_size!=None: - self.pool_size=pool_size - # if batch is not None, assign it to the batch attribute - if batch!=None: - self.batch=batch - # if update_step is not None, assign it to the update_step attribute - if update_step!=None: - self.update_step=update_step - # if trial_count is not None, assign it to the trial_count attribute - if trial_count!=None: - self.trial_count=trial_count - # if criterion is not None, assign it to the criterion attribute - if criterion!=None: - self.criterion=criterion - # call the action_vec method - self.action_vec() - # return nothing - return - - # define a method named epsilon_greedy_policy - def epsilon_greedy_policy(self,s): - # assign a numpy array of ones with the length of action_count and multiply it by epsilon divided by the length of action_one to the action_prob variable - action_prob=self.action_one*self.epsilon/len(self.action_one) - # try to execute the following statements - try: - # assign the index of the maximum value of the output of the nn attribute's fp method with s as input to the best_a variable - best_a=np.argmax(self.nn.nn.fp(s)) - # add 1 minus epsilon to the best_a element of action_prob - action_prob[best_a]+=1-self.epsilon - # if an exception occurs, execute the following statements - except Exception as e: - # if the platform attribute has a DType attribute, raise the exception - if hasattr(self.platform,'DType'): - raise e - # else, assign the index of the maximum value of the output of the nn attribute's nn method with s as input to the best_a variable - else: - best_a=self.nn.nn(s).argmax() - # add 1 minus epsilon to the best_a element of action_prob and convert it to numpy array - action_prob[best_a.numpy()]+=1-self.epsilon - # return action_prob - return action_prob - - # define a method named get_reward - def get_reward(self,max_step=None,seed=None): - # initialize the reward variable as 0 - reward=0 - # if seed is None, assign the output of the genv attribute's reset method to the s variable - if seed==None: - s=self.genv.reset() - # else, assign the output of the genv attribute's reset method with seed as input to the s variable + # else, assign the index of the maximum value of the output of the nn attribute's nn method with s as input to the best_a variable else: - s=self.genv.reset(seed=seed) - # if max_step is not None, execute the following statements - if max_step!=None: - # use a for loop to iterate from 0 to max_step - for i in range(max_step): - # if the end_flag attribute is True, break the loop - if self.end_flag==True: - break - # if the nn attribute has a nn attribute, execute the following statements - if hasattr(self.nn,'nn'): + best_a=self.nn.nn(s).argmax() + # add 1 minus epsilon to the best_a element of action_prob and convert it to numpy array + action_prob[best_a.numpy()]+=1-self.epsilon + # return action_prob + return action_prob + + # define a method named get_reward + def get_reward(self,max_step=None,seed=None): + # initialize the reward variable as 0 + reward=0 + # if seed is None, assign the output of the genv attribute's reset method to the s variable + if seed==None: + s=self.genv.reset() + # else, assign the output of the genv attribute's reset method with seed as input to the s variable + else: + s=self.genv.reset(seed=seed) + # if max_step is not None, execute the following statements + if max_step!=None: + # use a for loop to iterate from 0 to max_step + for i in range(max_step): + # if the end_flag attribute is True, break the loop + if self.end_flag==True: + break + # if the nn attribute has a nn attribute, execute the following statements + if hasattr(self.nn,'nn'): + # if the platform attribute has a DType attribute, execute the following statements + if hasattr(self.platform,'DType'): + # expand the dimension of s along axis 0 and assign it back to s + s=np.expand_dims(s,axis=0) + # assign the index of the maximum value of the output of the nn attribute's nn method with s as input to the a variable + a=np.argmax(self.nn.nn.fp(s)) + # else, execute the following statements + else: + # expand the dimension of s along axis 0 and assign it back to s + s=np.expand_dims(s,axis=0) + # convert s to a tensor with float dtype and assign it to the device attribute of the nn attribute and assign it back to s + s=self.platform.tensor(s,dtype=self.platform.float).to(self.nn.device) + # assign the index of the maximum value of the output of the nn attribute's nn method with s as input to the a variable and convert it to numpy array + a=self.nn.nn(s).detach().numpy().argmax() + # else, execute the following statements + else: + # if the nn attribute has an action attribute, execute the following statements + if hasattr(self.nn,'action'): # if the platform attribute has a DType attribute, execute the following statements if hasattr(self.platform,'DType'): # expand the dimension of s along axis 0 and assign it back to s s=np.expand_dims(s,axis=0) - # assign the index of the maximum value of the output of the nn attribute's nn method with s as input to the a variable - a=np.argmax(self.nn.nn.fp(s)) + # assign the output of the nn attribute's action method with s as input to the a variable and convert it to numpy array + a=self.nn.action(s).numpy() # else, execute the following statements else: # expand the dimension of s along axis 0 and assign it back to s s=np.expand_dims(s,axis=0) # convert s to a tensor with float dtype and assign it to the device attribute of the nn attribute and assign it back to s s=self.platform.tensor(s,dtype=self.platform.float).to(self.nn.device) - # assign the index of the maximum value of the output of the nn attribute's nn method with s as input to the a variable and convert it to numpy array - a=self.nn.nn(s).detach().numpy().argmax() + # assign the output of the nn attribute's action method with s as input to the a variable and convert it to numpy array and detach it from computation graph + a=self.nn.action(s).detach().numpy() # else, execute the following statements else: - # if the nn attribute has an action attribute, execute the following statements - if hasattr(self.nn,'action'): - # if the platform attribute has a DType attribute, execute the following statements - if hasattr(self.platform,'DType'): - # expand the dimension of s along axis 0 and assign it back to s - s=np.expand_dims(s,axis=0) - # assign the output of the nn attribute's action method with s as input to the a variable and convert it to numpy array - a=self.nn.action(s).numpy() - # else, execute the following statements - else: - # expand the dimension of s along axis 0 and assign it back to s - s=np.expand_dims(s,axis=0) - # convert s to a tensor with float dtype and assign it to the device attribute of the nn attribute and assign it back to s - s=self.platform.tensor(s,dtype=self.platform.float).to(self.nn.device) - # assign the output of the nn attribute's action method with s as input to the a variable and convert it to numpy array and detach it from computation graph - a=self.nn.action(s).detach().numpy() - # else, execute the following statements - else: - # if the platform attribute has a DType attribute, execute the following statements - if hasattr(self.platform,'DType'): - # expand the dimension of s along axis 0 and assign it back to s - s=np.expand_dims(s,axis=0) - # assign the output of adding up output of fp method from actor attribute of nn with noise from nn as input to a variable and convert it to numpy array - a=(self.nn.actor.fp(s)+self.nn.noise()).numpy() - # squeeze out any singleton dimensions from a and assign it back to a - a=np.squeeze(a) - # else, execute the following statements - else: - # expand the dimension of s along axis 0 and assign it back to s - s=np.expand_dims(s,axis=0) - # convert s to a tensor with float dtype and assign it to device attribute of nn and assign it back to s - s=self.platform.tensor(s,dtype=self.platform.float).to(self.nn.device) - # assign output of adding up output from actor method from nn with noise from nn as input to a variable and convert it to numpy array and detach it from computation graph - a=(self.nn.actor(s)+self.nn.noise()).detach().numpy() - # squeeze out any singleton dimensions from a and assign it back to a - a=np.squeeze(a) - # assign the output of the genv attribute's step method with a as input to the next_s, r, done variables - next_s,r,done,_=self.genv.step(a) - # assign next_s to s - s=next_s - # add r to reward and assign it back to reward - reward+=r - # if the nn attribute has a stop attribute, execute the following statements - if hasattr(self.nn,'stop'): - # if the output of the nn attribute's stop method with next_s as input is True, break the loop - if self.nn.stop(next_s): - break - # if done is True, break the loop - if done: - break - # return reward - return reward - # else, execute the following statements - else: - # use a while loop to iterate indefinitely - while True: - # if the end_flag attribute is True, break the loop - if self.end_flag==True: - break - # if the nn attribute has a nn attribute, execute the following statements - if hasattr(self.nn,'nn'): # if the platform attribute has a DType attribute, execute the following statements if hasattr(self.platform,'DType'): # expand the dimension of s along axis 0 and assign it back to s s=np.expand_dims(s,axis=0) - # assign the index of the maximum value of the output of the nn attribute's nn method with s as input to the a variable - a=np.argmax(self.nn.nn.fp(s)) - # else, execute the following statements + # assign the output of adding up output of fp method from actor attribute of nn with noise from nn as input to a variable and convert it to numpy array + a=(self.nn.actor.fp(s)+self.nn.noise()).numpy() + # squeeze out any singleton dimensions from a and assign it back to a + a=np.squeeze(a) + # else, execute the following statements else: - # expand the dimension of s along axis 0 and assign it back to s + # expand the dimension of s along axis 0 and assign it back to s s=np.expand_dims(s,axis=0) - # convert s to a tensor with float dtype and assign it to the device attribute of the nn attribute and assign it back to s + # convert s to a tensor with float dtype and assign it to device attribute of nn and assign it back to s s=self.platform.tensor(s,dtype=self.platform.float).to(self.nn.device) - # assign the index of the maximum value of the output of the nn attribute's nn method with s as input to the a variable and convert it to numpy array - a=self.nn.nn(s).detach().numpy().argmax() - # else, execute the following statements - else: - # if the nn attribute has an action attribute, execute the following statements - if hasattr(self.nn,'action'): - # if the platform attribute has a DType attribute, execute the following statements - if hasattr(self.platform,'DType'): - # expand the dimension of s along axis 0 and assign it back to s - s=np.expand_dims(s,axis=0) - # assign the output of the nn attribute's action method with s as input to the a variable and convert it to numpy array - a=self.nn.action(s).numpy() - # else, execute the following statements - else: - # expand the dimension of s along axis 0 and assign it back to s - s=np.expand_dims(s,axis=0) - # convert s to a tensor with float dtype and assign it to device attribute of nn and assign it back to s - s=self.platform.tensor(s,dtype=self.platform.float).to(self.nn.device) - # assign output of action method from nn with s as input to a variable and convert it to numpy array and detach it from computation graph - a=self.nn.action(s).detach().numpy() - # else, execute the following statements - else: - # if platform has DType attribute, execute following statements - if hasattr(self.platform,'DType'): - # expand dimension of s along axis 0 and assign it back to s - s=np.expand_dims(s,axis=0) - # assign output of adding up output from fp method from actor attribute of nn with noise from nn as input to a variable and convert it to numpy array - a=(self.nn.actor.fp(s)+self.nn.noise()).numpy() - # squeeze out any singleton dimensions from a and assign it back to a - a=np.squeeze(a) - # else, execute following statements - else: - # expand dimension of s along axis 0 and assign it back to s - s=np.expand_dims(s,axis=0) - # convert s to tensor with float dtype and assign it to device attribute of nn and assign it back to s - s=self.platform.tensor(s,dtype=self.platform.float).to(self.nn.device) - # assign output of adding up output from actor method from nn with noise from nn as input to a variable and convert it to numpy array and detach it from computation graph - a=(self.nn.actor(s)+self.nn.noise()).detach().numpy() - # squeeze out any singleton dimensions from a and assign it back to a - a=np.squeeze(a) - # assign output of genv attribute's step method with a as input to next_s, r, done variables - next_s,r,done,_=self.genv.step(a) - # assign next_s to s - s=next_s - # add r to reward and assign it back to reward - reward+=r - # if the nn attribute has a stop attribute, execute the following statements - if hasattr(self.nn,'stop'): - # if the output of the nn attribute's stop method with next_s as input is True, break the loop - if self.nn.stop(next_s): - break - # if done is True, break the loop - if done: + # assign output of adding up output from actor method from nn with noise from nn as input to a variable and convert it to numpy array and detach it from computation graph + a=(self.nn.actor(s)+self.nn.noise()).detach().numpy() + # squeeze out any singleton dimensions from a and assign it back to a + a=np.squeeze(a) + # assign the output of the genv attribute's step method with a as input to the next_s, r, done variables + next_s,r,done,_=self.genv.step(a) + # assign next_s to s + s=next_s + # add r to reward and assign it back to reward + reward+=r + # if the nn attribute has a stop attribute, execute the following statements + if hasattr(self.nn,'stop'): + # if the output of the nn attribute's stop method with next_s as input is True, break the loop + if self.nn.stop(next_s): break - # return reward - return reward - - # define a method named get_episode - def get_episode(self,max_step=None,seed=None): - # initialize the counter variable as 0 - counter=0 - # initialize the episode variable as an empty list - episode=[] - # if seed is None, assign the output of the genv attribute's reset method to the s variable - if seed==None: - s=self.genv.reset() - # else, assign the output of the genv attribute's reset method with seed as input to the s variable - else: - s=self.genv.reset(seed=seed) - # initialize the end_flag attribute as False - self.end_flag=False + # if done is True, break the loop + if done: + break + # return reward + return reward + # else, execute the following statements + else: # use a while loop to iterate indefinitely while True: # if the end_flag attribute is True, break the loop @@ -351,33 +261,33 @@ def get_episode(self,max_step=None,seed=None): s=np.expand_dims(s,axis=0) # assign the index of the maximum value of the output of the nn attribute's nn method with s as input to the a variable a=np.argmax(self.nn.nn.fp(s)) - # else, execute the following statements + # else, execute the following statements else: - # expand the dimension of s along axis 0 and assign it back to s + # expand the dimension of s along axis 0 and assign it back to s s=np.expand_dims(s,axis=0) - # convert s to a tensor with float dtype and assign it to the device attribute of the nn attribute and assign it back to s + # convert s to a tensor with float dtype and assign it to the device attribute of the nn attribute and assign it back to s s=self.platform.tensor(s,dtype=self.platform.float).to(self.nn.device) - # assign the index of the maximum value of the output of the nn attribute's nn method with s as input to the a variable and convert it to numpy array + # assign the index of the maximum value of the output of the nn attribute's nn method with s as input to the a variable and convert it to numpy array a=self.nn.nn(s).detach().numpy().argmax() - # else, execute the following statements + # else, execute the following statements else: - # if the nn attribute has an action attribute, execute the following statements + # if the nn attribute has an action attribute, execute the following statements if hasattr(self.nn,'action'): - # if the platform attribute has a DType attribute, execute the following statements + # if the platform attribute has a DType attribute, execute the following statements if hasattr(self.platform,'DType'): - # expand the dimension of s along axis 0 and assign it back to s + # expand the dimension of s along axis 0 and assign it back to s s=np.expand_dims(s,axis=0) - # assign output of action method from nn with s as input to a variable and convert it to numpy array + # assign the output of the nn attribute's action method with s as input to the a variable and convert it to numpy array a=self.nn.action(s).numpy() - # else, execute following statements + # else, execute the following statements else: - # expand dimension of s along axis 0 and assign it back to s + # expand the dimension of s along axis 0 and assign it back to s s=np.expand_dims(s,axis=0) - # convert s to tensor with float dtype and assign it to device attribute of nn and assign it back to s + # convert s to a tensor with float dtype and assign it to device attribute of nn and assign it back to s s=self.platform.tensor(s,dtype=self.platform.float).to(self.nn.device) # assign output of action method from nn with s as input to a variable and convert it to numpy array and detach it from computation graph a=self.nn.action(s).detach().numpy() - # else, execute following statements + # else, execute the following statements else: # if platform has DType attribute, execute following statements if hasattr(self.platform,'DType'): @@ -397,777 +307,867 @@ def get_episode(self,max_step=None,seed=None): a=(self.nn.actor(s)+self.nn.noise()).detach().numpy() # squeeze out any singleton dimensions from a and assign it back to a a=np.squeeze(a) - # assign the output of the genv attribute's step method with a as input to the next_s, r, done variables + # assign output of genv attribute's step method with a as input to next_s, r, done variables next_s,r,done,_=self.genv.step(a) - # if the end_flag attribute is True, break the loop - if self.end_flag==True: - break - # if done is True, execute the following statements - if done: - # append a list of s, a, next_s, r to the episode variable - episode.append([s,a,next_s,r]) - # append the string 'done' to the episode variable - episode.append('done') - # break the loop - break - # else, execute the following statements - else: - # append a list of s, a, next_s, r to the episode variable - episode.append([s,a,next_s,r]) - # if max_step is not None and counter equals max_step minus 1, break the loop - if max_step!=None and counter==max_step-1: - break # assign next_s to s s=next_s - # increment counter by 1 - counter+=1 - # return episode - return episode - - # define a method named tf_opt - @function(jit_compile=True) - def tf_opt(self,state_batch,action_batch,next_state_batch,reward_batch,done_batch): - # use the platform attribute's GradientTape method with persistent as True as a context manager and assign it to the tape variable - with self.platform.GradientTape(persistent=True) as tape: - # assign the output of the nn attribute's loss method with state_batch, action_batch, next_state_batch, reward_batch, done_batch as inputs to the loss variable - loss=self.nn.loss(state_batch,action_batch,next_state_batch,reward_batch,done_batch) - # if the nn attribute has a gradient attribute, execute the following statements - if hasattr(self.nn,'gradient'): - # assign the output of the nn attribute's gradient method with tape and loss as inputs to the gradient variable - gradient=self.nn.gradient(tape,loss) - # if the opt attribute of the nn attribute has an apply_gradients method, execute the following statements - if hasattr(self.nn.opt,'apply_gradients'): - # call the apply_gradients method of the opt attribute of the nn attribute with zip function of gradient and param attribute of nn as input - self.nn.opt.apply_gradients(zip(gradient,self.nn.param)) - # else, execute following statements + # add r to reward and assign it back to reward + reward+=r + # if the nn attribute has a stop attribute, execute the following statements + if hasattr(self.nn,'stop'): + # if the output of the nn attribute's stop method with next_s as input is True, break the loop + if self.nn.stop(next_s): + break + # if done is True, break the loop + if done: + break + # return reward + return reward + + # define a method named get_episode + def get_episode(self,max_step=None,seed=None): + # initialize the counter variable as 0 + counter=0 + # initialize the episode variable as an empty list + episode=[] + # if seed is None, assign the output of the genv attribute's reset method to the s variable + if seed==None: + s=self.genv.reset() + # else, assign the output of the genv attribute's reset method with seed as input to the s variable + else: + s=self.genv.reset(seed=seed) + # initialize the end_flag attribute as False + self.end_flag=False + # use a while loop to iterate indefinitely + while True: + # if the end_flag attribute is True, break the loop + if self.end_flag==True: + break + # if the nn attribute has a nn attribute, execute the following statements + if hasattr(self.nn,'nn'): + # if the platform attribute has a DType attribute, execute the following statements + if hasattr(self.platform,'DType'): + # expand the dimension of s along axis 0 and assign it back to s + s=np.expand_dims(s,axis=0) + # assign the index of the maximum value of the output of the nn attribute's nn method with s as input to the a variable + a=np.argmax(self.nn.nn.fp(s)) + # else, execute the following statements else: - # call opt attribute of nn with gradient as input - self.nn.opt(gradient) - # else, execute following statements + # expand the dimension of s along axis 0 and assign it back to s + s=np.expand_dims(s,axis=0) + # convert s to a tensor with float dtype and assign it to the device attribute of the nn attribute and assign it back to s + s=self.platform.tensor(s,dtype=self.platform.float).to(self.nn.device) + # assign the index of the maximum value of the output of the nn attribute's nn method with s as input to the a variable and convert it to numpy array + a=self.nn.nn(s).detach().numpy().argmax() + # else, execute the following statements else: - # if nn has nn attribute, execute following statements - if hasattr(self.nn,'nn'): - # assign output of tape's gradient method with loss and param attribute of nn as inputs to gradient variable - gradient=tape.gradient(loss,self.nn.param) - # call apply_gradients method of opt attribute of nn with zip function of gradient and param attribute of nn as input - self.nn.opt.apply_gradients(zip(gradient,self.nn.param)) + # if the nn attribute has an action attribute, execute the following statements + if hasattr(self.nn,'action'): + # if the platform attribute has a DType attribute, execute the following statements + if hasattr(self.platform,'DType'): + # expand the dimension of s along axis 0 and assign it back to s + s=np.expand_dims(s,axis=0) + # assign output of action method from nn with s as input to a variable and convert it to numpy array + a=self.nn.action(s).numpy() + # else, execute following statements + else: + # expand dimension of s along axis 0 and assign it back to s + s=np.expand_dims(s,axis=0) + # convert s to tensor with float dtype and assign it to device attribute of nn and assign it back to s + s=self.platform.tensor(s,dtype=self.platform.float).to(self.nn.device) + # assign output of action method from nn with s as input to a variable and convert it to numpy array and detach it from computation graph + a=self.nn.action(s).detach().numpy() # else, execute following statements else: - # assign output of tape's gradient method with first element of loss and first element of param attribute of nn as inputs to actor_gradient variable - actor_gradient=tape.gradient(loss[0],self.nn.param[0]) - # assign output of tape's gradient method with second element of loss and second element of param attribute of nn as inputs to critic_gradient variable - critic_gradient=tape.gradient(loss[1],self.nn.param[1]) - # call apply_gradients method of opt attribute of nn with zip function of actor_gradient and first element of param attribute of nn as input - self.nn.opt.apply_gradients(zip(actor_gradient,self.nn.param[0])) - # call apply_gradients method of opt attribute of nn with zip function of critic_gradient and second element of param attribute of nn as input - self.nn.opt.apply_gradients(zip(critic_gradient,self.nn.param[1])) - # return loss - return loss - - # define a method named pytorch_opt - def pytorch_opt(self,state_batch,action_batch,next_state_batch,reward_batch,done_batch): - # assign output of loss method from nn with state_batch, action_batch, next_state_batch, reward_batch, done_batch as inputs to loss variable - loss=self.nn.loss(state_batch,action_batch,next_state_batch,reward_batch,done_batch) - # assign next_state_batch to loss variable + # if platform has DType attribute, execute following statements + if hasattr(self.platform,'DType'): + # expand dimension of s along axis 0 and assign it back to s + s=np.expand_dims(s,axis=0) + # assign output of adding up output from fp method from actor attribute of nn with noise from nn as input to a variable and convert it to numpy array + a=(self.nn.actor.fp(s)+self.nn.noise()).numpy() + # squeeze out any singleton dimensions from a and assign it back to a + a=np.squeeze(a) + # else, execute following statements + else: + # expand dimension of s along axis 0 and assign it back to s + s=np.expand_dims(s,axis=0) + # convert s to tensor with float dtype and assign it to device attribute of nn and assign it back to s + s=self.platform.tensor(s,dtype=self.platform.float).to(self.nn.device) + # assign output of adding up output from actor method from nn with noise from nn as input to a variable and convert it to numpy array and detach it from computation graph + a=(self.nn.actor(s)+self.nn.noise()).detach().numpy() + # squeeze out any singleton dimensions from a and assign it back to a + a=np.squeeze(a) + # assign the output of the genv attribute's step method with a as input to the next_s, r, done variables + next_s,r,done,_=self.genv.step(a) + # if the end_flag attribute is True, break the loop + if self.end_flag==True: + break + # if done is True, execute the following statements + if done: + # append a list of s, a, next_s, r to the episode variable + episode.append([s,a,next_s,r]) + # append the string 'done' to the episode variable + episode.append('done') + # break the loop + break + # else, execute the following statements + else: + # append a list of s, a, next_s, r to the episode variable + episode.append([s,a,next_s,r]) + # if max_step is not None and counter equals max_step minus 1, break the loop + if max_step!=None and counter==max_step-1: + break + # assign next_s to s + s=next_s + # increment counter by 1 + counter+=1 + # return episode + return episode + + # define a method named tf_opt + @function(jit_compile=True) + def tf_opt(self,state_batch,action_batch,next_state_batch,reward_batch,done_batch): + # use the platform attribute's GradientTape method with persistent as True as a context manager and assign it to the tape variable + with self.platform.GradientTape(persistent=True) as tape: + # assign the output of the nn attribute's loss method with state_batch, action_batch, next_state_batch, reward_batch, done_batch as inputs to the loss variable loss=self.nn.loss(state_batch,action_batch,next_state_batch,reward_batch,done_batch) - # call zero_grad method of opt attribute of nn - self.nn.opt.zero_grad() + # if the nn attribute has a gradient attribute, execute the following statements + if hasattr(self.nn,'gradient'): + # assign the output of the nn attribute's gradient method with tape and loss as inputs to the gradient variable + gradient=self.nn.gradient(tape,loss) + # if the opt attribute of the nn attribute has an apply_gradients method, execute the following statements + if hasattr(self.nn.opt,'apply_gradients'): + # call the apply_gradients method of the opt attribute of the nn attribute with zip function of gradient and param attribute of nn as input + self.nn.opt.apply_gradients(zip(gradient,self.nn.param)) + # else, execute following statements + else: + # call opt attribute of nn with gradient as input + self.nn.opt(gradient) + # else, execute following statements + else: # if nn has nn attribute, execute following statements if hasattr(self.nn,'nn'): - # call backward method of loss - loss.backward() - # call step method of opt attribute of nn - self.nn.opt.step() + # assign output of tape's gradient method with loss and param attribute of nn as inputs to gradient variable + gradient=tape.gradient(loss,self.nn.param) + # call apply_gradients method of opt attribute of nn with zip function of gradient and param attribute of nn as input + self.nn.opt.apply_gradients(zip(gradient,self.nn.param)) # else, execute following statements else: - # call backward method of first element of loss - loss[0].backward() - # call step method of opt attribute of nn - self.nn.opt.step() - # call zero_grad method of opt attribute of nn - self.nn.opt.zero_grad() - # call backward method of second element of loss - loss[1].backward() - # call step method of opt attribute of nn - self.nn.opt.step() - # return loss - return loss - - # define a method named opt - def opt(self,state_batch,action_batch,next_state_batch,reward_batch,done_batch): - # if the platform attribute has a DType attribute, execute the following statements - if hasattr(self.platform,'DType'): - # call the tf_opt method with state_batch, action_batch, next_state_batch, reward_batch, done_batch as inputs and assign the output to the loss variable - loss=self.tf_opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch) - # else, execute the following statements - else: - # call the pytorch_opt method with state_batch, action_batch, next_state_batch, reward_batch, done_batch as inputs and assign the output to the loss variable - loss=self.pytorch_opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch) - # return loss - return loss - - # define a method named opt_ol - def opt_ol(self,state,action,next_state,reward,done): - # if the platform attribute has a DType attribute, execute the following statements - if hasattr(self.platform,'DType'): - # call the tf_opt method with state, action, next_state, reward, done as inputs and assign the output to loss variable - loss=self.tf_opt(state,action,next_state,reward,done) - # else, execute the following statements - else: - # call the pytorch_opt method with state, action, next_state, reward, done as inputs and assign the output to loss variable - loss=self.pytorch_opt(state,action,next_state,reward,done) - # return the loss variable - return loss - - # define a method named pool - def pool(self,s,a,next_s,r,done): - # if the state_pool attribute is None, execute the following statements - if self.state_pool==None: - # expand the dimension of s along axis 0 and assign it to the state_pool attribute - self.state_pool=np.expand_dims(s,axis=0) - # if a is an integer, execute the following statements - if type(a)==int: - # if the type of the first element of the param attribute of the nn attribute is not a list, execute the following statements - if type(self.nn.param[0])!=list: - # convert a to a numpy array with the dtype name of the first element of the param attribute of the nn attribute and assign it to a - a=np.array(a,self.nn.param[0].dtype.name) - # else, execute the following statements - else: - # convert a to a numpy array with the dtype name of the first element of the first element of the param attribute of the nn attribute and assign it to a - a=np.array(a,self.nn.param[0][0].dtype.name) - # expand the dimension of a along axis 0 and assign it to the action_pool attribute - self.action_pool=np.expand_dims(a,axis=0) + # assign output of tape's gradient method with first element of loss and first element of param attribute of nn as inputs to actor_gradient variable + actor_gradient=tape.gradient(loss[0],self.nn.param[0]) + # assign output of tape's gradient method with second element of loss and second element of param attribute of nn as inputs to critic_gradient variable + critic_gradient=tape.gradient(loss[1],self.nn.param[1]) + # call apply_gradients method of opt attribute of nn with zip function of actor_gradient and first element of param attribute of nn as input + self.nn.opt.apply_gradients(zip(actor_gradient,self.nn.param[0])) + # call apply_gradients method of opt attribute of nn with zip function of critic_gradient and second element of param attribute of nn as input + self.nn.opt.apply_gradients(zip(critic_gradient,self.nn.param[1])) + # return loss + return loss + + # define a method named pytorch_opt + def pytorch_opt(self,state_batch,action_batch,next_state_batch,reward_batch,done_batch): + # assign output of loss method from nn with state_batch, action_batch, next_state_batch, reward_batch, done_batch as inputs to loss variable + loss=self.nn.loss(state_batch,action_batch,next_state_batch,reward_batch,done_batch) + # assign next_state_batch to loss variable + loss=self.nn.loss(state_batch,action_batch,next_state_batch,reward_batch,done_batch) + # call zero_grad method of opt attribute of nn + self.nn.opt.zero_grad() + # if nn has nn attribute, execute following statements + if hasattr(self.nn,'nn'): + # call backward method of loss + loss.backward() + # call step method of opt attribute of nn + self.nn.opt.step() + # else, execute following statements + else: + # call backward method of first element of loss + loss[0].backward() + # call step method of opt attribute of nn + self.nn.opt.step() + # call zero_grad method of opt attribute of nn + self.nn.opt.zero_grad() + # call backward method of second element of loss + loss[1].backward() + # call step method of opt attribute of nn + self.nn.opt.step() + # return loss + return loss + + # define a method named opt + def opt(self,state_batch,action_batch,next_state_batch,reward_batch,done_batch): + # if the platform attribute has a DType attribute, execute the following statements + if hasattr(self.platform,'DType'): + # call the tf_opt method with state_batch, action_batch, next_state_batch, reward_batch, done_batch as inputs and assign the output to the loss variable + loss=self.tf_opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch) + # else, execute the following statements + else: + # call the pytorch_opt method with state_batch, action_batch, next_state_batch, reward_batch, done_batch as inputs and assign the output to the loss variable + loss=self.pytorch_opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch) + # return loss + return loss + + # define a method named opt_ol + def opt_ol(self,state,action,next_state,reward,done): + # if the platform attribute has a DType attribute, execute the following statements + if hasattr(self.platform,'DType'): + # call the tf_opt method with state, action, next_state, reward, done as inputs and assign the output to loss variable + loss=self.tf_opt(state,action,next_state,reward,done) + # else, execute the following statements + else: + # call the pytorch_opt method with state, action, next_state, reward, done as inputs and assign the output to loss variable + loss=self.pytorch_opt(state,action,next_state,reward,done) + # return the loss variable + return loss + + # define a method named pool + def pool(self,s,a,next_s,r,done): + # if the state_pool attribute is None, execute the following statements + if self.state_pool==None: + # expand the dimension of s along axis 0 and assign it to the state_pool attribute + self.state_pool=np.expand_dims(s,axis=0) + # if a is an integer, execute the following statements + if type(a)==int: + # if the type of the first element of the param attribute of the nn attribute is not a list, execute the following statements + if type(self.nn.param[0])!=list: + # convert a to a numpy array with the dtype name of the first element of the param attribute of the nn attribute and assign it to a + a=np.array(a,self.nn.param[0].dtype.name) # else, execute the following statements else: - # if the type of the first element of the param attribute of the nn attribute is not a list, execute the following statements - if type(self.nn.param[0])!=list: - # convert a to its dtype name with the first element of the param attribute of the nn attribute and assign it back to a - a=a.astype(self.nn.param[0].dtype.name) - # else, execute the following statements - else: - # convert a to its dtype name with the first element of the first element of the param attribute of the nn attribute and assign it back to a - a=a.astype(self.nn.param[0][0].dtype.name) - # assign a to the action_pool attribute - self.action_pool=a - # expand the dimension of next_s along axis 0 and assign it to the next_state_pool attribute - self.next_state_pool=np.expand_dims(next_s,axis=0) - # expand the dimension of r along axis 0 and assign it to the reward_pool attribute - self.reward_pool=np.expand_dims(r,axis=0) - # expand the dimension of done along axis 0 and assign it to the done_pool attribute - self.done_pool=np.expand_dims(done,axis=0) + # convert a to a numpy array with the dtype name of the first element of the first element of the param attribute of the nn attribute and assign it to a + a=np.array(a,self.nn.param[0][0].dtype.name) + # expand the dimension of a along axis 0 and assign it to the action_pool attribute + self.action_pool=np.expand_dims(a,axis=0) # else, execute the following statements else: - # append s to the state_pool attribute along axis 0 - self.state_pool=np.append(self.state_pool,np.expand_dims(s,axis=0),axis=0) - # if a is an integer, execute the following statements - if type(a)==int: - # if the type of the first element of the param attribute of the nn attribute is not a list, execute the following statements - if type(self.nn.param[0])!=list: - # convert a to a numpy array with the dtype name of the first element of the param attribute of the nn attribute and assign it to a - a=np.array(a,self.nn.param[0].dtype.name) - # else, execute the following statements - else: - # convert a to a numpy array with the dtype name of the first element of the first element of the param attribute of the nn attribute and assign it to a - a=np.array(a,self.nn.param[0][0].dtype.name) - # append a to the action_pool attribute along axis 0 - self.action_pool=np.append(self.action_pool,np.expand_dims(a,axis=0),axis=0) + # if the type of the first element of the param attribute of the nn attribute is not a list, execute the following statements + if type(self.nn.param[0])!=list: + # convert a to its dtype name with the first element of the param attribute of the nn attribute and assign it back to a + a=a.astype(self.nn.param[0].dtype.name) # else, execute the following statements else: - # if the type of the first element of the param attribute of the nn attribute is not a list, execute the following statements - if type(self.nn.param[0])!=list: - # convert a to its dtype name with the first element of the param attribute of the nn attribute and assign it back to a - a=a.astype(self.nn.param[0].dtype.name) - # else, execute the following statements - else: - # convert a to its dtype name with the first element of the first element of the param attribute of the nn attribute and assign it back to a - a=a.astype(self.nn.param[0][0].dtype.name) - # append a to the action_pool attribute along axis 0 - self.action_pool=np.append(self.action_pool,a,axis=0) - # append next_s to the next_state_pool attribute along axis 0 - self.next_state_pool=np.append(self.next_state_pool,np.expand_dims(next_s,axis=0),axis=0) - # append r to the reward_pool attribute along axis 0 - self.reward_pool=np.append(self.reward_pool,np.expand_dims(r,axis=0),axis=0) - # append done to the done_pool attribute along axis 0 - self.done_pool=np.append(self.done_pool,np.expand_dims(done,axis=0),axis=0) - # return nothing - return - - # define a method named choose_action - def choose_action(self,s): - # if hasattr(self.platform,'DType'): - if hasattr(self.platform,'DType'): - # expand dimension of s along axis 0 and assign it back to s - s=np.expand_dims(s,axis=0) - # assign output from actor method from nn with s as input to a variable and convert it to numpy array - a=(self.nn.actor.fp(s)+self.nn.noise()).numpy() - # else, execute following statements - else: - # expand dimension of s along axis 0 and assign it back to s - s=np.expand_dims(s,axis=0) - # convert s to tensor with float dtype and assign it to device attribute of nn and assign it back to s - s=self.platform.tensor(s,dtype=self.platform.float).to(self.nn.device) - # assign output from actor method from nn with s as input to a variable and convert it to numpy array and detach it from computation graph - a=(self.nn.actor(s)+self.nn.noise()).detach().numpy() - # if hasattr(self.nn,'discriminator'): - if hasattr(self.nn,'discriminator'): - # assign output from discriminator method from nn with s as input to reward variable and convert it to numpy array - reward=self.nn.discriminator(s).numpy() - # else, assign None to reward variable - else: - reward=None - # return a and reward - return a,reward - - # define a method named _train - def _train(self): - # if length of state_pool is less than batch, return numpy array of 0 - if len(self.state_pool)self.pool_size: - # assign a numpy array of 0 to the TD variable - TD=np.array(0) - # append TD to the TD attribute of the pr attribute of the nn attribute from index 2 onwards - self.nn.pr.TD=np.append(TD,self.nn.pr.TD[2:]) - # add r to the reward attribute and assign it back to the reward attribute - self.reward=r+self.reward - # assign the output of the _train method to the loss variable - loss=self._train() - # increment the sc attribute by 1 - self.sc+=1 - # if done is True, execute the following statements - if done: - # if the save_episode attribute is True, execute the following statements - if self.save_episode==True: - # append a list of s, a, next_s, r to the episode variable - episode=[s,a,next_s,r] - # append the reward attribute to the reward_list attribute - self.reward_list.append(self.reward) - # return loss, episode, done - return loss,episode,done - # else if save_episode is True, execute following statements - elif self.save_episode==True: - # append list of s, a, next_s, r to episode variable - episode=[s,a,next_s,r] - # assign next_s to s - s=next_s + # assign output from data_func method from nn with state_pool, action_pool, next_state_pool, reward_pool, done_pool, batch as inputs to state_batch, action_batch, next_state_batch, reward_batch, done_batch variables + state_batch,action_batch,next_state_batch,reward_batch,done_batch=self.nn.data_func(self.state_pool,self.action_pool,self.next_state_pool,self.reward_pool,self.done_pool,self.batch) + # assign output from opt method with state_batch, action_batch, next_state_batch, reward_batch, done_batch as inputs to batch_loss variable + batch_loss=self.opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch) + # add batch_loss to loss and assign it back to loss + loss+=batch_loss + # if nn has bc attribute, execute following statements + if hasattr(self.nn,'bc'): + # try to execute following statements + try: + # call assign_add method of bc attribute of nn with 1 as input + self.nn.bc.assign_add(1) + # if exception occurs, execute following statements + except Exception: + # increment bc attribute of nn by 1 + self.nn.bc+=1 + # if remainder of dividing length of state_pool by batch is not 0, execute following statements + if len(self.state_pool)%self.batch!=0: + # call suspend_func method + self.suspend_func() + # assign output from data_func method from nn with state_pool, action_pool, next_state_pool, reward_pool, done_pool, batch as inputs to state_batch, action_batch, next_state_batch, reward_batch, done_batch variables + state_batch,action_batch,next_state_batch,reward_batch,done_batch=self.nn.data_func(self.state_pool,self.action_pool,self.next_state_pool,self.reward_pool,self.done_pool,self.batch) + # assign output from opt method with state_batch, action_batch, next_state_batch, reward_batch, done_batch as inputs to batch_loss variable + batch_loss=self.opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch) + # add batch_loss to loss and assign it back to loss + loss+=batch_loss + # if nn has bc attribute, execute following statements + if hasattr(self.nn,'bc'): + # try to execute following statements + try: + # call assign_add method of bc attribute of nn with 1 as input + self.nn.bc.assign_add(1) + # if exception occurs, execute following statements + except Exception: + # increment bc attribute of nn by 1 + self.nn.bc+=1 # else, execute following statements else: - # use while loop to iterate indefinitely - while True: + # initialize j variable as 0 + j=0 + # assign output from Dataset method from tf_data module with tensor_slices method with state_pool, action_pool, next_state_pool, reward_pool, done_pool as input and shuffle method with length of state_pool as input and batch method with batch as input to train_ds variable + train_ds=tf_data.Dataset.from_tensor_slices((self.state_pool,self.action_pool,self.next_state_pool,self.reward_pool,self.done_pool)).shuffle(len(self.state_pool)).batch(self.batch) + # use for loop to iterate over train_ds and assign outputs to state_batch, action_batch, next_state_batch, reward_batch, done_batch variables + for state_batch,action_batch,next_state_batch,reward_batch,done_batch in train_ds: # call suspend_func method self.suspend_func() - # assign output from epsilon_greedy_policy method with s as input to action_prob variable - action_prob=self.epsilon_greedy_policy(s) - # assign random choice from range of action_count with action_prob as probability to a variable - a=np.random.choice(range(self.action_count),p=action_prob) - # if nn has env attribute, execute following statements - if hasattr(self.nn,'env'): - # assign output from env method from nn with a as input to next_s, r, done variables - next_s,r,done=self.nn.env(a) - # else, execute following statements - else: - # assign output from step method from genv with a as input to next_s, r, done variables - next_s,r,done,_=self.genv.step(a) - # if hasattr(self.platform,'DType'): - if hasattr(self.platform,'DType'): - # if type of first element of param attribute of nn is not list, execute following statements - if type(self.nn.param[0])!=list: - # convert next_s to numpy array with dtype name of first element of param attribute of nn and assign it back to next_s - next_s=np.array(next_s,self.nn.param[0].dtype.name) - # convert r to numpy array with dtype name of first element of param attribute of nn and assign it back to r - r=np.array(r,self.nn.param[0].dtype.name) - # convert done to numpy array with dtype name of first element of param attribute of nn and assign it back to done - done=np.array(done,self.nn.param[0].dtype.name) - # else, execute following statements - else: - # convert next_s to numpy array with dtype name of first element of first element of param attribute of nn and assign it back to next_s - next_s=np.array(next_s,self.nn.param[0][0].dtype.name) - # convert r to numpy array with dtype name of first element of first element of param attribute of nn and assign it back to r - r=np.array(r,self.nn.param[0][0].dtype.name) - # convert done to numpy array with dtype name of first element of first element of param attribute of nn and assign it back to done - done=np.array(done,self.nn.param[0][0].dtype.name) - # if the nn attribute has a pool method, execute the following statements - if hasattr(self.nn,'pool'): - # call the pool method of the nn attribute with the state_pool, action_pool, next_state_pool, reward_pool, done_pool, s, a, next_s, r, done as inputs - self.nn.pool(self.state_pool,self.action_pool,self.next_state_pool,self.reward_pool,self.done_pool,[s,a,next_s,r,done]) - # else, execute the following statements - else: - # call the pool method with s, a, next_s, r, done as inputs - self.pool(s,a,next_s,r,done) - # if the nn attribute has a pr attribute, execute the following statements - if hasattr(self.nn,'pr'): - # append the initial_TD attribute of the nn attribute to the TD attribute of the pr attribute of the nn attribute - self.nn.pr.TD=np.append(self.nn.pr.TD,self.nn.initial_TD) - # if the length of the state_pool attribute is greater than the pool_size attribute, execute the following statements - if len(self.state_pool)>self.pool_size: - # assign a numpy array of 0 to the TD variable - TD=np.array(0) - # append TD to the TD attribute of the pr attribute of the nn attribute from index 2 onwards - self.nn.pr.TD=np.append(TD,self.nn.pr.TD[2:]) - # add r to the reward attribute and assign it back to the reward attribute - self.reward=r+self.reward - # assign the output of the _train method to the loss variable - loss=self._train() - # increment the sc attribute by 1 - self.sc+=1 - # if done is True or save_episode is True, execute the following statements - if done or self.save_episode==True: - # initialize episode variable as an empty list - episode=[] - # use a for loop to iterate over range from 0 to sc plus 1 - for i in range(0,self.sc+1): - # append a list of state_pool[i], action_pool[i], next_state_pool[i], reward_pool[i] to episode variable - episode.append([self.state_pool[i],self.action_pool[i],self.next_state_pool[i],self.reward_pool[i]]) - # if done is True, append 'done' string to episode variable - if done: - episode.append('done') - # return loss, episode, done - return loss,episode,done - # assign next_s to s - s=next_s - - # define a method named train - def train(self,episode_count,save=None,one=True,p=None,s=None): - # initialize the average reward variable as None - avg_reward=None - # if p is None, assign 9 to the p attribute - if p==None: - self.p=9 - # else, assign p minus 1 to the p attribute - else: - self.p=p-1 - # if s is None, assign 1 to the s attribute and None to the file_list attribute - if s==None: - self.s=1 - self.file_list=None - # else, assign s minus 1 to the s attribute and an empty list to the file_list attribute + # assign output from opt method with state_batch, action_batch, next_state_batch, reward_batch, done_batch as inputs to batch_loss variable + batch_loss=self.opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch) + # add batch_loss to loss and assign it back to loss + loss+=batch_loss + # if nn has bc attribute, execute following statements + if hasattr(self.nn,'bc'): + # try to execute following statements + try: + # call assign_add method of bc attribute of nn with 1 as input + self.nn.bc.assign_add(1) + # if exception occurs, execute following statements + except Exception: + # increment bc attribute of nn by 1 + self.nn.bc+=1 + # increment j by 1 + j+=1 + # divide loss by j and assign it back to loss + loss=loss/j + # return loss + return loss + + # define a method named train_ + def train_(self): + # initialize the reward attribute as 0 + self.reward=0 + # initialize the episode variable as None + episode=None + # if the nn attribute has a reset method, execute the following statements + if hasattr(self.nn,'reset'): + # call the reset method of the nn attribute + self.nn.reset() + # if the nn attribute has a env attribute, execute the following statements + if hasattr(self.nn,'env'): + # assign the output of the env attribute's reset method to the s variable + s=self.nn.env.reset() + # else, execute the following statements else: - self.s=s-1 - self.file_list=[] - # if episode_count is not None, execute the following statements - if episode_count!=None: - # use a for loop to iterate from 0 to episode_count - for i in range(episode_count): - # record the current time as t1 - t1=time.time() - # call the train_ method and assign the outputs to loss, episode, and done variables - loss,episode,done=self.train_() - # if the trial_count attribute is not None, execute the following statements - if self.trial_count!=None: - # if the length of the reward_list attribute is greater than or equal to the trial_count attribute, execute the following statements - if len(self.reward_list)>=self.trial_count: - # calculate the mean of the last trial_count elements of the reward_list attribute and assign it to avg_reward - avg_reward=statistics.mean(self.reward_list[-self.trial_count:]) - # if the criterion attribute is not None and avg_reward is greater than or equal to the criterion attribute, execute the following statements - if self.criterion!=None and avg_reward>=self.criterion: - # record the current time as t2 - t2=time.time() - # add t2 minus t1 to the total_time attribute and assign it back to itself - self.total_time+=(t2-t1) - # assign total_time minus its integer part to time_ attribute - time_=self.total_time-int(self.total_time) - # if time_ is less than 0.5, assign the integer part of total_time to itself - if time_<0.5: - self.total_time=int(self.total_time) - # else, assign the integer part of total_time plus 1 to itself - else: - self.total_time=int(self.total_time)+1 - # print the total_episode attribute - print('episode:{0}'.format(self.total_episode)) - # print the loss with six decimal places - print('last loss:{0:.6f}'.format(loss)) - # print the avg_reward - print('average reward:{0}'.format(avg_reward)) - print() - # print the total_time with 's' as unit - print('time:{0}s'.format(self.total_time)) - # return nothing - return - # assign the loss to the loss attribute - self.loss=loss - # append the loss to the loss_list attribute - self.loss_list.append(loss) - # increment the total_episode attribute by 1 - self.total_episode+=1 - # if episode_count is not None and episode_count is divisible by 10, execute the following statements - if episode_count!=None and episode_count%10==0: - # assign episode_count divided by p plus 1 to p and convert it to an integer - p=int(episode_count/(self.p+1)) - # assign episode_count divided by s plus 1 to s and convert it to an integer - s=int(episode_count/(self.s+1)) + # assign the output of the genv attribute's reset method to the s variable + s=self.genv.reset() + # if the episode_step attribute is not None, execute the following statements + if self.episode_step!=None: + # use a for loop to iterate from 0 to episode_step + for _ in range(self.episode_step): + # call the suspend_func method + self.suspend_func() + # assign the output of the epsilon_greedy_policy method with s as input to the action_prob variable + action_prob=self.epsilon_greedy_policy(s) + # assign a random choice from range of action_count with action_prob as probability to the a variable + a=np.random.choice(range(self.action_count),p=action_prob) + # if the nn attribute has a env attribute, execute the following statements + if hasattr(self.nn,'env'): + # assign the output of the env attribute's step method with a as input to the next_s, r, done variables + next_s,r,done=self.nn.env(a) # else, execute the following statements else: - # assign episode_count minus its remainder when divided by p to p and divide it by p and convert it to an integer - p=int((episode_count-episode_count%self.p)/self.p) - # assign episode_count minus its remainder when divided by s to s and divide it by s and convert it to an integer - s=int((episode_count-episode_count%s)/self.s) - # if p is 0, assign 1 to p - if p==0: - p=1 - # if s is 0, assign 1 to s - if s==0: - s=1 - # if i plus 1 is divisible by p, execute the following statements - if (i+1)%p==0: - # if the length of the state_pool attribute is greater than or equal to the batch attribute, execute the following statements - if len(self.state_pool)>=self.batch: - # print i plus 1 and loss with six decimal places - print('episode:{0} loss:{1:.6f}'.format(i+1,loss)) - # if avg_reward is not None, execute the following statements - if avg_reward!=None: - # print i plus 1 and avg_reward - print('episode:{0} average reward:{1}'.format(i+1,avg_reward)) - # else, execute the following statements + # assign the output of the genv attribute's step method with a as input to the next_s, r, done variables + next_s,r,done,_=self.genv.step(a) + # if hasattr(self.platform,'DType'): + if hasattr(self.platform,'DType'): + # if type of first element of param attribute of nn is not list, execute following statements + if type(self.nn.param[0])!=list: + # convert next_s to numpy array with dtype name of first element of param attribute of nn and assign it back to next_s + next_s=np.array(next_s,self.nn.param[0].dtype.name) + # convert r to numpy array with dtype name of first element of param attribute of nn and assign it back to r + r=np.array(r,self.nn.param[0].dtype.name) + # convert done to numpy array with dtype name of first element of param attribute of nn and assign it back to done + done=np.array(done,self.nn.param[0].dtype.name) + # else, execute following statements else: - # print i plus 1 and reward - print('episode:{0} reward:{1}'.format(i+1,self.reward)) - print() - # if save is not None and i plus 1 is divisible by s, execute the following statements - if save!=None and (i+1)%s==0: - # call the save method with total_episode and one as inputs - self.save(self.total_episode,one) - # if save_episode is True, execute the following statements - if self.save_episode==True: - # append episode to the episode_set attribute - self.episode_set.append(episode) - # if max_episode_count is not None and length of episode_set attribute is greater than or equal to max_episode_count attribute, execute the following statements - if self.max_episode_count!=None and len(self.episode_set)>=self.max_episode_count: - # assign False to save_episode attribute - self.save_episode=False - try: - try: - # call the assign_add method of the ec attribute of the nn attribute with 1 as input - self.nn.ec.assign_add(1) - except Exception: - # increment the ec attribute of the nn attribute by 1 - self.nn.ec+=1 - except Exception: - pass - t2=time.time() - self.time+=(t2-t1) - # else, execute the following statements + # convert next_s to numpy array with dtype name of first element of first element of param attribute of nn and assign it back to next_s + next_s=np.array(next_s,self.nn.param[0][0].dtype.name) + # convert r to numpy array with dtype name of first element of first element of param attribute of nn and assign it back to r + r=np.array(r,self.nn.param[0][0].dtype.name) + # convert done to numpy array with dtype name of first element of first element of param attribute of nn and assign it back to done + done=np.array(done,self.nn.param[0][0].dtype.name) + # if the nn attribute has a pool method, execute the following statements + if hasattr(self.nn,'pool'): + # call the pool method of the nn attribute with the state_pool, action_pool, next_state_pool, reward_pool, done_pool, s, a, next_s, r, done as inputs + self.nn.pool(self.state_pool,self.action_pool,self.next_state_pool,self.reward_pool,self.done_pool,[s,a,next_s,r,done]) + # else, execute the following statements + else: + # call the pool method with s, a, next_s, r, done as inputs + self.pool(s,a,next_s,r,done) + # if the nn attribute has a pr attribute, execute the following statements + if hasattr(self.nn,'pr'): + # append the initial_TD attribute of the nn attribute to the TD attribute of the pr attribute of the nn attribute + self.nn.pr.TD=np.append(self.nn.pr.TD,self.nn.initial_TD) + # if the length of the state_pool attribute is greater than the pool_size attribute, execute the following statements + if len(self.state_pool)>self.pool_size: + # assign a numpy array of 0 to the TD variable + TD=np.array(0) + # append TD to the TD attribute of the pr attribute of the nn attribute from index 2 onwards + self.nn.pr.TD=np.append(TD,self.nn.pr.TD[2:]) + # add r to the reward attribute and assign it back to the reward attribute + self.reward=r+self.reward + # assign the output of the _train method to the loss variable + loss=self._train() + # increment the sc attribute by 1 + self.sc+=1 + # if done is True, execute the following statements + if done: + # if the save_episode attribute is True, execute the following statements + if self.save_episode==True: + # append a list of s, a, next_s, r to the episode variable + episode=[s,a,next_s,r] + # append the reward attribute to the reward_list attribute + self.reward_list.append(self.reward) + # return loss, episode, done + return loss,episode,done + # else if save_episode is True, execute following statements + elif self.save_episode==True: + # append list of s, a, next_s, r to episode variable + episode=[s,a,next_s,r] + # assign next_s to s + s=next_s + # else, execute following statements else: - # initialize i as 0 - i=0 - # use a while loop to iterate indefinitely + # use while loop to iterate indefinitely while True: - # record the current time as t1 - t1=time.time() - # call the train_ method and assign the outputs to loss, episode, and done variables - loss,episode,done=self.train_() - # if the trial_count attribute is not None, execute the following statements - if self.trial_count!=None: - # if the length of the reward_list attribute is equal to the trial_count attribute, execute the following statements - if len(self.reward_list)==self.trial_count: - # calculate the mean of the last trial_count elements of the reward_list attribute and assign it to avg_reward - avg_reward=statistics.mean(self.reward_list[-self.trial_count:]) - # if avg_reward is greater than or equal to the criterion attribute, execute the following statements - if avg_reward>=self.criterion: - # record the current time as t2 - t2=time.time() - # add t2 minus t1 to the total_time attribute and assign it back to itself - self.total_time+=(t2-t1) - # assign total_time minus its integer part to time_ attribute - time_=self.total_time-int(self.total_time) - # if time_ is less than 0.5, assign the integer part of total_time to itself - if time_<0.5: - self.total_time=int(self.total_time) - # else, assign the integer part of total_time plus 1 to itself - else: - self.total_time=int(self.total_time)+1 - # print the total_episode attribute - print('episode:{0}'.format(self.total_episode)) - # print the loss with six decimal places - print('last loss:{0:.6f}'.format(loss)) - # print the avg_reward - print('average reward:{0}'.format(avg_reward)) - print() - # print the total_time with 's' as unit - print('time:{0}s'.format(self.total_time)) - # return nothing - return - # assign the loss to the loss attribute - self.loss=loss - # append the loss to the loss_list attribute - self.loss_list.append(loss) - # increment i by 1 - i+=1 - # increment the total_episode attribute by 1 - self.total_episode+=1 - # if episode_count is not None and episode_count is not divisible by 10, execute the following statements - if episode_count!=None and episode_count%10!=0: - # assign episode_count minus its remainder when divided by p to p and divide it by p and convert it to an integer - p=int((episode_count-episode_count%self.p)/self.p) - # assign episode_count minus its remainder when divided by s to s and divide it by s and convert it to an integer - s=int((episode_count-episode_count%s)/self.s) - # else, execute the following statements + # call suspend_func method + self.suspend_func() + # assign output from epsilon_greedy_policy method with s as input to action_prob variable + action_prob=self.epsilon_greedy_policy(s) + # assign random choice from range of action_count with action_prob as probability to a variable + a=np.random.choice(range(self.action_count),p=action_prob) + # if nn has env attribute, execute following statements + if hasattr(self.nn,'env'): + # assign output from env method from nn with a as input to next_s, r, done variables + next_s,r,done=self.nn.env(a) + # else, execute following statements else: - # assign episode_count divided by p plus 1 to p and convert it to an integer - p=int(episode_count/(self.p+1)) - # assign episode_count divided by s plus 1 to s and convert it to an integer - s=int(episode_count/(self.s+1)) - # if p is 0, assign 1 to p - if p==0: - p=1 - # if s is 0, assign 1 to s - if s==0: - s=1 - # if i plus 1 is divisible by p, execute the following statements - if (i+1)%p==0: - # if the length of the state_pool attribute is greater than or equal to the batch attribute, execute the following statements - if len(self.state_pool)>=self.batch: - # print i plus 1 and loss with six decimal places - print('episode:{0} loss:{1:.6f}'.format(i+1,loss)) - # if avg_reward is not None, execute the following statements - if avg_reward!=None: - # print i plus 1 and avg_reward - print('episode:{0} average reward:{1}'.format(i+1,avg_reward)) - # else, execute the following statements + # assign output from step method from genv with a as input to next_s, r, done variables + next_s,r,done,_=self.genv.step(a) + # if hasattr(self.platform,'DType'): + if hasattr(self.platform,'DType'): + # if type of first element of param attribute of nn is not list, execute following statements + if type(self.nn.param[0])!=list: + # convert next_s to numpy array with dtype name of first element of param attribute of nn and assign it back to next_s + next_s=np.array(next_s,self.nn.param[0].dtype.name) + # convert r to numpy array with dtype name of first element of param attribute of nn and assign it back to r + r=np.array(r,self.nn.param[0].dtype.name) + # convert done to numpy array with dtype name of first element of param attribute of nn and assign it back to done + done=np.array(done,self.nn.param[0].dtype.name) + # else, execute following statements else: - # print i plus 1 and reward - print('episode:{0} reward:{1}'.format(i+1,self.reward)) - print() - # if save is not None and i plus 1 is divisible by s, execute the following statements - if save!=None and (i+1)%s==0: - # call the save method with total_episode and one as inputs - self.save(self.total_episode,one) - # if save_episode is True, execute the following statements - if self.save_episode==True: - # if done is True, append 'done' string to episode + # convert next_s to numpy array with dtype name of first element of first element of param attribute of nn and assign it back to next_s + next_s=np.array(next_s,self.nn.param[0][0].dtype.name) + # convert r to numpy array with dtype name of first element of first element of param attribute of nn and assign it back to r + r=np.array(r,self.nn.param[0][0].dtype.name) + # convert done to numpy array with dtype name of first element of first element of param attribute of nn and assign it back to done + done=np.array(done,self.nn.param[0][0].dtype.name) + # if the nn attribute has a pool method, execute the following statements + if hasattr(self.nn,'pool'): + # call the pool method of the nn attribute with the state_pool, action_pool, next_state_pool, reward_pool, done_pool, s, a, next_s, r, done as inputs + self.nn.pool(self.state_pool,self.action_pool,self.next_state_pool,self.reward_pool,self.done_pool,[s,a,next_s,r,done]) + # else, execute the following statements + else: + # call the pool method with s, a, next_s, r, done as inputs + self.pool(s,a,next_s,r,done) + # if the nn attribute has a pr attribute, execute the following statements + if hasattr(self.nn,'pr'): + # append the initial_TD attribute of the nn attribute to the TD attribute of the pr attribute of the nn attribute + self.nn.pr.TD=np.append(self.nn.pr.TD,self.nn.initial_TD) + # if the length of the state_pool attribute is greater than the pool_size attribute, execute the following statements + if len(self.state_pool)>self.pool_size: + # assign a numpy array of 0 to the TD variable + TD=np.array(0) + # append TD to the TD attribute of the pr attribute of the nn attribute from index 2 onwards + self.nn.pr.TD=np.append(TD,self.nn.pr.TD[2:]) + # add r to the reward attribute and assign it back to the reward attribute + self.reward=r+self.reward + # assign the output of the _train method to the loss variable + loss=self._train() + # increment the sc attribute by 1 + self.sc+=1 + # if done is True or save_episode is True, execute the following statements + if done or self.save_episode==True: + # initialize episode variable as an empty list + episode=[] + # use a for loop to iterate over range from 0 to sc plus 1 + for i in range(0,self.sc+1): + # append a list of state_pool[i], action_pool[i], next_state_pool[i], reward_pool[i] to episode variable + episode.append([self.state_pool[i],self.action_pool[i],self.next_state_pool[i],self.reward_pool[i]]) + # if done is True, append 'done' string to episode variable if done: episode.append('done') - # append episode to the episode_set attribute - self.episode_set.append(episode) - # if max_episode_count is not None and length of episode_set attribute is greater than or equal to max_episode_count attribute, execute the following statements - if self.max_episode_count!=None and len(self.episode_set)>=self.max_episode_count: - # assign False to save_episode attribute - self.save_episode=False - # if the nn attribute has an ec attribute, execute the following statements - if hasattr(self.nn,'ec'): - # try to execute the following statements - try: - # call the assign_add method of the ec attribute of the nn attribute with 1 as input - self.nn.ec.assign_add(1) - # if an exception occurs, execute the following statements - except Exception: - # increment the ec attribute of the nn attribute by 1 - self.nn.ec+=1 - # record the current time as t2 - t2=time.time() - # add t2 minus t1 to the time attribute and assign it back to itself - self.time+=(t2-t1) - # assign time minus its integer part to time_ attribute - time_=self.time-int(self.time) - # if time_ is less than 0.5, assign the integer part of time to itself - if time_<0.5: - self.total_time=int(self.time) - # else, assign the integer part of time plus 1 to itself - else: - self.total_time=int(self.time)+1 - # add time to the total_time attribute and assign it back to itself - self.total_time+=self.time - # print the loss with six decimal places - print('last loss:{0:.6f}'.format(loss)) - # print the reward - print('last reward:{0}'.format(self.reward)) - print() - # print the time with 's' as unit - print('time:{0}s'.format(self.time)) - # return nothing - return - - # define a method named train_online - def train_online(self): - # use while loop to iterate indefinitely - while True: - # if hasattr(nn,'save'): - if hasattr(self.nn,'save'): - # call save method of nn with save as input - self.nn.save(self.save) - # if hasattr(nn,'stop_flag'): - if hasattr(self.nn,'stop_flag'): - # if stop_flag attribute of nn is True, return nothing - if self.nn.stop_flag==True: + # return loss, episode, done + return loss,episode,done + # assign next_s to s + s=next_s + +# define a method named train +def train(self,episode_count,save=None,one=True,p=None,s=None): + # initialize the average reward variable as None + avg_reward=None + # if p is None, assign 9 to the p attribute + if p==None: + self.p=9 + # else, assign p minus 1 to the p attribute + else: + self.p=p-1 + # if s is None, assign 1 to the s attribute and None to the file_list attribute + if s==None: + self.s=1 + self.file_list=None + # else, assign s minus 1 to the s attribute and an empty list to the file_list attribute + else: + self.s=s-1 + self.file_list=[] + # if episode_count is not None, execute the following statements + if episode_count!=None: + # use a for loop to iterate from 0 to episode_count + for i in range(episode_count): + # record the current time as t1 + t1=time.time() + # call the train_ method and assign the outputs to loss, episode, and done variables + loss,episode,done=self.train_() + # if the trial_count attribute is not None, execute the following statements + if self.trial_count!=None: + # if the length of the reward_list attribute is greater than or equal to the trial_count attribute, execute the following statements + if len(self.reward_list)>=self.trial_count: + # calculate the mean of the last trial_count elements of the reward_list attribute and assign it to avg_reward + avg_reward=statistics.mean(self.reward_list[-self.trial_count:]) + # if the criterion attribute is not None and avg_reward is greater than or equal to the criterion attribute, execute the following statements + if self.criterion!=None and avg_reward>=self.criterion: + # record the current time as t2 + t2=time.time() + # add t2 minus t1 to the total_time attribute and assign it back to itself + self.total_time+=(t2-t1) + # assign total_time minus its integer part to time_ attribute + time_=self.total_time-int(self.total_time) + # if time_ is less than 0.5, assign the integer part of total_time to itself + if time_<0.5: + self.total_time=int(self.total_time) + # else, assign the integer part of total_time plus 1 to itself + else: + self.total_time=int(self.total_time)+1 + # print the total_episode attribute + print('episode:{0}'.format(self.total_episode)) + # print the loss with six decimal places + print('last loss:{0:.6f}'.format(loss)) + # print the avg_reward + print('average reward:{0}'.format(avg_reward)) + print() + # print the total_time with 's' as unit + print('time:{0}s'.format(self.total_time)) + # return nothing return - # if hasattr(nn,'stop_func'): - if hasattr(self.nn,'stop_func'): - # if output of stop_func method of nn is True, return nothing - if self.nn.stop_func(): + # assign the loss to the loss attribute + self.loss=loss + # append the loss to the loss_list attribute + self.loss_list.append(loss) + # increment the total_episode attribute by 1 + self.total_episode+=1 + # if episode_count is not None and episode_count is divisible by 10, execute the following statements + if episode_count!=None and episode_count%10==0: + # assign episode_count divided by p plus 1 to p and convert it to an integer + p=int(episode_count/(self.p+1)) + # assign episode_count divided by s plus 1 to s and convert it to an integer + s=int(episode_count/(self.s+1)) + # else, execute the following statements + else: + # assign episode_count minus its remainder when divided by p to p and divide it by p and convert it to an integer + p=int((episode_count-episode_count%self.p)/self.p) + # assign episode_count minus its remainder when divided by s to s and divide it by s and convert it to an integer + s=int((episode_count-episode_count%s)/self.s) + # if p is 0, assign 1 to p + if p==0: + p=1 + # if s is 0, assign 1 to s + if s==0: + s=1 + # if i plus 1 is divisible by p, execute the following statements + if (i+1)%p==0: + # if the length of the state_pool attribute is greater than or equal to the batch attribute, execute the following statements + if len(self.state_pool)>=self.batch: + # print i plus 1 and loss with six decimal places + print('episode:{0} loss:{1:.6f}'.format(i+1,loss)) + # if avg_reward is not None, execute the following statements + if avg_reward!=None: + # print i plus 1 and avg_reward + print('episode:{0} average reward:{1}'.format(i+1,avg_reward)) + # else, execute the following statements + else: + # print i plus 1 and reward + print('episode:{0} reward:{1}'.format(i+1,self.reward)) + print() + # if save is not None and i plus 1 is divisible by s, execute the following statements + if save!=None and (i+1)%s==0: + # call the save method with total_episode and one as inputs + self.save(self.total_episode,one) + # if save_episode is True, execute the following statements + if self.save_episode==True: + # append episode to the episode_set attribute + self.episode_set.append(episode) + # if max_episode_count is not None and length of episode_set attribute is greater than or equal to max_episode_count attribute, execute the following statements + if self.max_episode_count!=None and len(self.episode_set)>=self.max_episode_count: + # assign False to save_episode attribute + self.save_episode=False + try: + try: + # call the assign_add method of the ec attribute of the nn attribute with 1 as input + self.nn.ec.assign_add(1) + except Exception: + # increment the ec attribute of the nn attribute by 1 + self.nn.ec+=1 + except Exception: + pass + t2=time.time() + self.time+=(t2-t1) + # else, execute the following statements + else: + # initialize i as 0 + i=0 + # use a while loop to iterate indefinitely + while True: + # record the current time as t1 + t1=time.time() + # call the train_ method and assign the outputs to loss, episode, and done variables + loss,episode,done=self.train_() + # if the trial_count attribute is not None, execute the following statements + if self.trial_count!=None: + # if the length of the reward_list attribute is equal to the trial_count attribute, execute the following statements + if len(self.reward_list)==self.trial_count: + # calculate the mean of the last trial_count elements of the reward_list attribute and assign it to avg_reward + avg_reward=statistics.mean(self.reward_list[-self.trial_count:]) + # if avg_reward is greater than or equal to the criterion attribute, execute the following statements + if avg_reward>=self.criterion: + # record the current time as t2 + t2=time.time() + # add t2 minus t1 to the total_time attribute and assign it back to itself + self.total_time+=(t2-t1) + # assign total_time minus its integer part to time_ attribute + time_=self.total_time-int(self.total_time) + # if time_ is less than 0.5, assign the integer part of total_time to itself + if time_<0.5: + self.total_time=int(self.total_time) + # else, assign the integer part of total_time plus 1 to itself + else: + self.total_time=int(self.total_time)+1 + # print the total_episode attribute + print('episode:{0}'.format(self.total_episode)) + # print the loss with six decimal places + print('last loss:{0:.6f}'.format(loss)) + # print the avg_reward + print('average reward:{0}'.format(avg_reward)) + print() + # print the total_time with 's' as unit + print('time:{0}s'.format(self.total_time)) + # return nothing return - # if hasattr(nn,'suspend_func'): - if hasattr(self.nn,'suspend_func'): - # call suspend_func method of nn - self.nn.suspend_func() - # assign output from online method of nn to data variable - data=self.nn.online() - # if data is 'stop', return nothing - if data=='stop': + # assign the loss to the loss attribute + self.loss=loss + # append the loss to the loss_list attribute + self.loss_list.append(loss) + # increment i by 1 + i+=1 + # increment the total_episode attribute by 1 + self.total_episode+=1 + # if episode_count is not None and episode_count is not divisible by 10, execute the following statements + if episode_count!=None and episode_count%10!=0: + # assign episode_count minus its remainder when divided by p to p and divide it by p and convert it to an integer + p=int((episode_count-episode_count%self.p)/self.p) + # assign episode_count minus its remainder when divided by s to s and divide it by s and convert it to an integer + s=int((episode_count-episode_count%s)/self.s) + # else, execute the following statements + else: + # assign episode_count divided by p plus 1 to p and convert it to an integer + p=int(episode_count/(self.p+1)) + # assign episode_count divided by s plus 1 to s and convert it to an integer + s=int(episode_count/(self.s+1)) + # if p is 0, assign 1 to p + if p==0: + p=1 + # if s is 0, assign 1 to s + if s==0: + s=1 + # if i plus 1 is divisible by p, execute the following statements + if (i+1)%p==0: + # if the length of the state_pool attribute is greater than or equal to the batch attribute, execute the following statements + if len(self.state_pool)>=self.batch: + # print i plus 1 and loss with six decimal places + print('episode:{0} loss:{1:.6f}'.format(i+1,loss)) + # if avg_reward is not None, execute the following statements + if avg_reward!=None: + # print i plus 1 and avg_reward + print('episode:{0} average reward:{1}'.format(i+1,avg_reward)) + # else, execute the following statements + else: + # print i plus 1 and reward + print('episode:{0} reward:{1}'.format(i+1,self.reward)) + print() + # if save is not None and i plus 1 is divisible by s, execute the following statements + if save!=None and (i+1)%s==0: + # call the save method with total_episode and one as inputs + self.save(self.total_episode,one) + # if save_episode is True, execute the following statements + if self.save_episode==True: + # if done is True, append 'done' string to episode + if done: + episode.append('done') + # append episode to the episode_set attribute + self.episode_set.append(episode) + # if max_episode_count is not None and length of episode_set attribute is greater than or equal to max_episode_count attribute, execute the following statements + if self.max_episode_count!=None and len(self.episode_set)>=self.max_episode_count: + # assign False to save_episode attribute + self.save_episode=False + # if the nn attribute has an ec attribute, execute the following statements + if hasattr(self.nn,'ec'): + # try to execute the following statements + try: + # call the assign_add method of the ec attribute of the nn attribute with 1 as input + self.nn.ec.assign_add(1) + # if an exception occurs, execute the following statements + except Exception: + # increment the ec attribute of the nn attribute by 1 + self.nn.ec+=1 + # record the current time as t2 + t2=time.time() + # add t2 minus t1 to the time attribute and assign it back to itself + self.time+=(t2-t1) + # assign time minus its integer part to time_ attribute + time_=self.time-int(self.time) + # if time_ is less than 0.5, assign the integer part of time to itself + if time_<0.5: + self.total_time=int(self.time) + # else, assign the integer part of time plus 1 to itself + else: + self.total_time=int(self.time)+1 + # add time to the total_time attribute and assign it back to itself + self.total_time+=self.time + # print the loss with six decimal places + print('last loss:{0:.6f}'.format(loss)) + # print the reward + print('last reward:{0}'.format(self.reward)) + print() + # print the time with 's' as unit + print('time:{0}s'.format(self.time)) + # return nothing + return + + # define a method named train_online + def train_online(self): + # use while loop to iterate indefinitely + while True: + # if hasattr(nn,'save'): + if hasattr(self.nn,'save'): + # call save method of nn with save as input + self.nn.save(self.save) + # if hasattr(nn,'stop_flag'): + if hasattr(self.nn,'stop_flag'): + # if stop_flag attribute of nn is True, return nothing + if self.nn.stop_flag==True: + return + # if hasattr(nn,'stop_func'): + if hasattr(self.nn,'stop_func'): + # if output of stop_func method of nn is True, return nothing + if self.nn.stop_func(): return - # elif data is 'suspend', execute following statements - elif data=='suspend': - # call suspend_func method of nn - self.nn.suspend_func() - # assign output from opt_ol method with first, second, third, fourth, fifth elements of data as inputs to loss variable - loss=self.opt_ol(data[0],data[1],data[2],data[3],data[4]) - # convert loss to numpy array and assign it back to loss - loss=loss.numpy() - # append loss to train_loss_list attribute of nn - self.nn.train_loss_list.append(loss) - # if length of train_acc_list attribute of nn equals max_length attribute of nn, delete first element of train_acc_list attribute of nn - if len(self.nn.train_acc_list)==self.nn.max_length: - del self.nn.train_acc_list[0] - # if hasattr(nn,'counter'): - if hasattr(self.nn,'counter'): - # increment counter attribute of nn by 1 - self.nn.counter+=1 - # return nothing - return + # if hasattr(nn,'suspend_func'): + if hasattr(self.nn,'suspend_func'): + # call suspend_func method of nn + self.nn.suspend_func() + # assign output from online method of nn to data variable + data=self.nn.online() + # if data is 'stop', return nothing + if data=='stop': + return + # elif data is 'suspend', execute following statements + elif data=='suspend': + # call suspend_func method of nn + self.nn.suspend_func() + # assign output from opt_ol method with first, second, third, fourth, fifth elements of data as inputs to loss variable + loss=self.opt_ol(data[0],data[1],data[2],data[3],data[4]) + # convert loss to numpy array and assign it back to loss + loss=loss.numpy() + # append loss to train_loss_list attribute of nn + self.nn.train_loss_list.append(loss) + # if length of train_acc_list attribute of nn equals max_length attribute of nn, delete first element of train_acc_list attribute of nn + if len(self.nn.train_acc_list)==self.nn.max_length: + del self.nn.train_acc_list[0] + # if hasattr(nn,'counter'): + if hasattr(self.nn,'counter'): + # increment counter attribute of nn by 1 + self.nn.counter+=1 + # return nothing + return From 200af299d977bb1f2bd2e4b5f6beceefb56f0ecc Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Sun, 23 Jul 2023 09:50:11 +0800 Subject: [PATCH 076/337] Add files via upload --- .../DL/kernel/process/kernel_pytorch.py | 462 ++++++++++++++++++ 1 file changed, 462 insertions(+) create mode 100644 Note 7.0 documentation/DL/kernel/process/kernel_pytorch.py diff --git a/Note 7.0 documentation/DL/kernel/process/kernel_pytorch.py b/Note 7.0 documentation/DL/kernel/process/kernel_pytorch.py new file mode 100644 index 00000000..699e53fe --- /dev/null +++ b/Note 7.0 documentation/DL/kernel/process/kernel_pytorch.py @@ -0,0 +1,462 @@ +import torch # import the torch library +from multiprocessing import Value,Array # import the Value and Array classes from the multiprocessing module +import numpy as np # import the numpy library + + +class kernel: # define a class named kernel + def __init__(self,nn=None): # define the constructor method of the class + self.nn=nn # assign the nn argument to the self.nn attribute + if hasattr(self.nn,'km'): # check if the nn attribute has an attribute named km + self.nn.km=1 # set the km attribute of the nn attribute to 1 + self.process=None # initialize the process attribute to None + self.process_t=None # initialize the process_t attribute to None + self.train_ds=None # initialize the train_ds attribute to None + self.batches_t=None # initialize the batches_t attribute to None + self.shuffle=False # initialize the shuffle attribute to False + self.priority_flag=False # initialize the priority_flag attribute to False + self.priority_p=0 # initialize the priority_p attribute to 0 + self.max_opt=None # initialize the max_opt attribute to None + self.epoch=None # initialize the epoch attribute to None + self.epoch_counter=0 # initialize the epoch_counter attribute to 0 + self.stop=False # initialize the stop attribute to False + self.stop_flag=False # initialize the stop_flag attribute to False + self.save_flag=False # initialize the save_flag attribute to False + self.save_epoch=None # initialize the save_epoch attribute to None + self.batch=None # initialize the batch attribute to None + self.epoch_=0 # initialize the epoch_ attribute to 0 + self.end_loss=None # initialize the end_loss attribute to None + self.end_acc=None # initialize the end_acc attribute to None + self.end_test_loss=None # initialize the end_test_loss attribute to None + self.end_test_acc=None # initialize the end_test_acc attribute to None + self.acc_flag='%' # initialize the acc_flag attribute to '%' + self.opt_counter=None # initialize the opt_counter attribute to None + self.p=None # initialize the p attribute to None + self.s=None # initialize the s attribute to None + self.saving_one=True # initialize the saving_one attribute to True + self.filename='save.dat' # initialize the filename attribute to 'save.dat' + self.train_loss=0 # initialize the train_loss attribute to 0 + self.train_acc=0 # initialize the train_acc attribute to 0 + self.train_loss_list=[] # initialize the train_loss_list attribute to an empty list + self.train_acc_list=[] # initialize the train_acc_list attribute to an empty list + self.test_loss=0 # initialize the test_loss attribute to 0 + self.test_acc=0 # initialize the test_acc attribute to 0 + self.test_loss_list=[] # initialize the test_loss_list attribute to an empty list + self.test_acc_list=[] # initialize the test_acc_list attribute to an empty list + self.test_flag=False # initialize the test_flag attribute to False + self.total_epoch=0 # initialize the total_epoch attribute to 0 + + + def data(self,train_dataset=None,test_dataset=None): + '''define a method named data that takes two arguments: train_dataset and test_dataset''' + + '''This method is used to assign and process the train_dataset and test_dataset attributes, + and also create some arrays for storing some statistics''' + + '''The train_dataset and test_dataset arguments are expected to be PyTorch Dataset objects or lists of PyTorch Dataset objects''' + + '''The train_dataset argument is used for training the neural network''' + + '''The test_dataset argument is used for testing or evaluating the neural network, and it is optional''' + + '''If the test_dataset argument is given, the test_flag attribute will be set to True, + indicating that the test method will be called after each epoch''' + + self.train_dataset=train_dataset # assign the train_dataset argument to the self.train_dataset attribute + self.test_dataset=test_dataset # assign the test_dataset argument to the self.test_dataset attribute + if test_dataset is not None: # check if the test_dataset argument is not None + self.test_flag=True # set the test_flag attribute to True + self.batch_counter=np.zeros(self.process,dtype=np.int32) # create an array of zeros with a length equal to the process attribute and an integer data type, and assign it to the batch_counter attribute + self.total_loss=np.zeros(self.process) # create an array of zeros with a length equal to the process attribute and a float data type, and assign it to the total_loss attribute + if hasattr(self.nn,'accuracy'): # check if the nn attribute has a method named accuracy + if type(self.nn.param[0])!=list: # check if the first element of the param attribute of the nn attribute is not a list + self.total_acc=np.zeros(self.process,dtype=self.nn.param[0].dtype.name) # create an array of zeros with a length equal to the process attribute and a data type equal to the data type of the first element of the param attribute of the nn attribute, and assign it to the total_acc attribute + else: # otherwise + self.total_acc=np.zeros(self.process,dtype=self.nn.param[0][0].dtype.name) # create an array of zeros with a length equal to the process attribute and a data type equal to the data type of the first element of the first element of the param attribute of the nn attribute, and assign it to the total_acc attribute + if self.priority_flag==True: # check if the priority_flag attribute is True + self.opt_counter=np.zeros(self.process,dtype=np.int32) # create an array of zeros with a length equal to the process attribute and an integer data type, and assign it to the opt_counter attribute + return # return nothing + + + def init(self,manager): + '''define a method named init that takes one argument: manager''' + + '''This method is used to initialize some shared variables for multiprocessing, + using the manager argument as a multiprocessing.Manager object''' + + '''The manager argument is expected to be a multiprocessing.Manager object that can create shared variables across processes''' + + self.epoch_counter=Value('i',self.epoch_counter) # create a shared variable with an integer data type and an initial value equal to the epoch_counter attribute, and assign it to the epoch_counter attribute + self.batch_counter=Array('i',self.batch_counter) # create a shared array with an integer data type and an initial value equal to the batch_counter attribute, and assign it to the batch_counter attribute + self.total_loss=Array('f',self.total_loss) # create a shared array with a float data type and an initial value equal to the total_loss attribute, and assign it to the total_loss attribute + self.total_epoch=Value('i',self.total_epoch) # create a shared variable with an integer data type and an initial value equal to the total_epoch attribute, and assign it to the total_epoch attribute + self.train_loss=Value('f',self.train_loss) # create a shared variable with a float data type and an initial value equal to the train_loss attribute, and assign it to the train_loss attribute + self.train_loss_list=manager.list(self.train_loss_list) # create a shared list with an initial value equal to the train_loss_list attribute, using the list method of the manager argument, and assign it to the train_loss_list attribute + self.priority_p=Value('i',self.priority_p) # create a shared variable with an integer data type and an initial value equal to the priority_p attribute, and assign it to the priority_p attribute + if self.test_flag==True: # check if the test_flag attribute is True + self.test_loss=Value('f',self.test_loss) # create a shared variable with a float data type and an initial value equal to the test_loss attribute, and assign it to the test_loss attribute + self.test_loss_list=manager.list(self.test_loss_list) # create a shared list with an initial value equal to the test_loss_list attribute, using the list method of the manager argument, and assign it to the test_loss_list attribute + if hasattr(self.nn,'accuracy'): # check if the nn attribute has a method named accuracy + self.total_acc=Array('f',self.total_acc) # create a shared array with a float data type and an initial value equal to the total_acc attribute, and assign it to the total_acc attribute + self.train_acc=Value('f',self.train_acc) # create a shared variable with a float data type and an initial value equal to the train_acc attribute, and assign it to the train_acc attribute + self.train_acc_list=manager.list(self.train_acc_list) # create a shared list with an initial value equal to the train_acc_list attribute, using the list method of the manager argument, and assign it to the train_acc_list attribute + if self.test_flag==True: # check if the test_flag attribute is True + self.test_acc=Value('f',self.test_acc) # create a shared variable with a float data type and an initial value equal to the test_acc attribute, and assign it to the test_acc attribute + self.test_acc_list=manager.list(self.test_acc_list) # create a shared list with an initial value equal to the test_acc_list attribute, using the list method of the manager argument, and assign it to the test_acc_list attribute + if self.priority_flag==True: # check if the priority_flag attribute is True + self.opt_counter=Array('i',self.opt_counter) # create a shared array with an integer data type and an initial value equal to the opt_counter attribute, and assign it to the opt_counter attribute + try: + self.nn.opt_counter=manager.list([self.nn.opt_counter]) # try to create a shared list with an initial value equal to a list containing the opt_counter attribute of the nn attribute, using the list method of the manager argument, and assign it to the opt_counter attribute of the nn attribute + except Exception: # handle any exception that may occur + self.opt_counter_=manager.list() # create an empty shared list using the list method of the manager argument, and assign it to the opt_counter_ attribute + try: + self.nn.ec=manager.list([self.nn.ec]) # try to create a shared list with an initial value equal to a list containing the ec attribute of the nn attribute, using the list method of the manager argument, and assign it to the ec attribute of the nn attribute + except Exception: # handle any exception that may occur + self.ec_=manager.list() # create an empty shared list using the list method of the manager argument, and assign it to the ec_ attribute + try: + self.nn.bc=manager.list([self.nn.bc]) # try to create a shared list with an initial value equal to a list containing the bc attribute of the nn attribute, using the list method of the manager argument, and assign it to the bc attribute of the nn attribute + except Exception: # handle any exception that may occur + self.bc_=manager.list() # create an empty shared list using the list method of the manager argument, and assign it to the bc_ attribute + self.epoch_=Value('i',self.epoch_) # create a shared variable with an integer data type and an initial value equal to the epoch_ attribute, and assign it to the epoch_ attribute + self.stop_flag=Value('b',self.stop_flag) # create a shared variable with a boolean data type and an initial value equal to the stop_flag attribute, and assign it to the stop_flag attribute + self.save_flag=Value('b',self.save_flag) # create a shared variable with a boolean data type and an initial value equal to the save_flag attribute, and assign it to the save_flag attribute + self.file_list=manager.list([]) # create an empty shared list using the list method of the manager argument, and assign it to the file_list attribute + return # return nothing + + + def init_online(self,manager): + '''define a method named init_online that takes one argument: manager''' + + '''This method is used to initialize some shared variables for online training, + using the manager argument as a multiprocessing.Manager object''' + + '''The manager argument is expected to be a multiprocessing.Manager object that can create shared variables across processes''' + + self.nn.train_loss_list=manager.list([]) # create an empty shared list using the list method of the manager argument, and assign it to the train_loss_list attribute of the nn attribute + self.nn.train_acc_list=manager.list([]) # create an empty shared list using the list method of the manager argument, and assign it to the train_acc_list attribute of the nn attribute + self.nn.counter=manager.list([]) # create an empty shared list using the list method of the manager argument, and assign it to the counter attribute of the nn attribute + self.nn.exception_list=manager.list([]) # create an empty shared list using the list method of the manager argument, and assign it to the exception_list attribute of the nn attribute + return # return nothing + + + def end(self): + '''define a method named end that takes no arguments''' + + '''This method is used to check if the training process should be stopped based on some criteria''' + + '''The criteria are based on some attributes of the class, such as end_loss, end_acc, end_test_loss, and end_test_acc''' + + '''These attributes are expected to be numbers that represent some thresholds for training loss, training accuracy, test loss, and test accuracy respectively''' + + '''The method will return a boolean value indicating whether the training process should be stopped or not''' + + if self.end_loss!=None and len(self.train_loss_list)!=0 and self.train_loss_list[-1]self.end_acc: # check if the end_acc attribute is not None and the train_acc_list attribute is not empty and the last element of the train_acc_list attribute is greater than the end_acc attribute + return True # return True + elif self.end_loss!=None and len(self.train_loss_list)!=0 and self.end_acc!=None and self.train_loss_list[-1]self.end_acc: # check if both previous conditions are met + return True # return True + elif self.end_test_loss!=None and len(self.test_loss_list)!=0 and self.test_loss_list[-1]self.end_test_acc: # check if the end_test_acc attribute is not None and the test_acc_list attribute is not empty and the last element of the test_acc_list attribute is greater than the end_test_acc attribute + return True # return True + elif self.end_test_loss!=None and len(self.test_loss_list)!=0 and self.end_test_acc!=None and self.test_loss_list[-1]self.end_test_acc: # check if both previous conditions are met + return True # return True + else: # otherwise + return False # return False + + + def opt_p(self,data,labels,p): + '''define a method named opt_p that takes three arguments: data, labels, and p''' + + '''This method is used to perform one step of optimization for the neural network, + using the data and labels arguments as the input and output of the neural network, + and the p argument as the index of the process''' + + '''The data and labels arguments are expected to be PyTorch tensors that represent a batch of input and output data for the neural network''' + + '''The p argument is expected to be an integer that represents the index of the process that calls this method''' + + try: # try to execute the following code block + try: # try to execute the following code block + try: # try to execute the following code block + output=self.nn.fp(data,p) # call the fp method of the nn attribute with the data and p arguments, and assign the return value to the output variable + loss=self.nn.loss(output,labels,p) # call the loss method of the nn attribute with the output, labels, and p arguments, and assign the return value to the loss variable + except Exception: # handle any exception that may occur + output,loss=self.nn.fp(data,labels,p) # call the fp method of the nn attribute with the data, labels, and p arguments, and assign the return values to the output and loss variables + except Exception: # handle any exception that may occur + try: # try to execute the following code block + output=self.nn.fp(data) # call the fp method of the nn attribute with the data argument, and assign the return value to the output variable + loss=self.nn.loss(output,labels) # call the loss method of the nn attribute with the output and labels arguments, and assign the return value to the loss variable + except Exception: # handle any exception that may occur + output,loss=self.nn.fp(data,labels) # call the fp method of the nn attribute with the data and labels arguments, and assign the return values to the output and loss variables + except Exception as e: # handle any exception that may occur + raise e # raise the exception again + if self.priority_flag==True and self.priority_p.value!=-1: # check if both priority_flag attribute is True and priority_p attribute is not -1 + while True: # enter an infinite loop + if p==self.priority_p.value: # check if p is equal to priority_p attribute + break # break out of the loop + else: # otherwise + continue # continue looping + self.nn.opt[p].zero_grad(set_to_none=True) # call the zero_grad method of the opt attribute of the nn attribute with the p argument, and set the set_to_none argument to True + loss=loss.clone() # create a copy of the loss variable and assign it to the loss variable + loss.backward() # call the backward method of the loss variable to compute the gradients + self.nn.opt[p].step() # call the step method of the opt attribute of the nn attribute with the p argument to update the parameters + return output,loss # return the output and loss variables + + + def opt(self,data,labels,p): + '''define a method named opt that takes three arguments: data, labels, and p''' + + '''This method is used to perform one step of optimization for the neural network, + using the data and labels arguments as the input and output of the neural network, + and the p argument as the index of the process''' + + '''The data and labels arguments are expected to be PyTorch tensors that represent a batch of input and output data for the neural network''' + + '''The p argument is expected to be an integer that represents the index of the process that calls this method''' + + output,loss=self.opt_p(data,labels,p) # call the opt_p method with the data, labels, and p arguments, and assign the return values to the output and loss variables + return output,loss # return the output and loss variables + + + def train7(self,train_loader,p,test_batch,lock): + '''define a method named train7 that takes four arguments: train_loader, p, test_batch, and lock''' + + '''This method is used to train the neural network for one epoch using a given train_loader, + using the p argument as the index of the process, + using the test_batch argument as a batch of test data for evaluation, + and using the lock argument as a multiprocessing.Lock object for synchronization''' + + '''The train_loader argument is expected to be a PyTorch DataLoader object that provides batches of training data for the neural network''' + + '''The p argument is expected to be an integer that represents the index of the process that calls this method''' + + '''The test_batch argument is expected to be a PyTorch tensor that represents a batch of test data for evaluation''' + + '''The lock argument is expected to be a multiprocessing.Lock object that can be used to acquire and release a lock for synchronization across processes''' + while True: # enter an infinite loop + for data_batch,labels_batch in train_loader: # iterate over the batches of training data from the train_loader + if hasattr(self.nn,'data_func'): # check if the nn attribute has a method named data_func + data_batch,labels_batch=self.nn.data_func(data_batch,labels_batch) # call the data_func method of the nn attribute with the data_batch and labels_batch arguments, and assign the return values to the data_batch and labels_batch variables + if self.priority_flag==True: # check if the priority_flag attribute is True + self.priority_p.value=np.argmax(self.opt_counter) # set the value of the priority_p attribute to the index of the maximum element of the opt_counter attribute + if self.max_opt!=None and self.opt_counter[self.priority_p.value]>=self.max_opt: # check if both max_opt attribute is not None and the element of the opt_counter attribute at the index of the priority_p attribute is greater than or equal to the max_opt attribute + self.priority_p.value=int(self.priority_p.value) # cast the value of the priority_p attribute to an integer + elif self.max_opt==None: # check if max_opt attribute is None + self.priority_p.value=int(self.priority_p.value) # cast the value of the priority_p attribute to an integer + else: # otherwise + self.priority_p.value=-1 # set the value of the priority_p attribute to -1 + if self.priority_flag==True: # check if the priority_flag attribute is True + self.opt_counter[p]=0 # set the element of the opt_counter attribute at the index of p to 0 + if hasattr(self.nn,'attenuate'): # check if the nn attribute has a method named attenuate + opt_counter=self.nn.opt_counter[0] # assign the first element of the opt_counter attribute of the nn attribute to the opt_counter variable + opt_counter[p]=0 # set the element of the opt_counter variable at the index of p to 0 + self.nn.opt_counter[0]=opt_counter # assign the opt_counter variable to the first element of the opt_counter attribute of the nn attribute + output,batch_loss=self.opt(data_batch,labels_batch,p) # call the opt method with the data_batch, labels_batch, and p arguments, and assign the return values to the output and batch_loss variables + if self.priority_flag==True: # check if the priority_flag attribute is True + opt_counter=np.frombuffer(self.opt_counter.get_obj(),dtype='i') # create a numpy array from a buffer object that contains a copy of memory from another object, using an integer data type, and assign it to the opt_counter variable + opt_counter+=1 # increment each element of the opt_counter variable by 1 + if hasattr(self.nn,'attenuate'): # check if the nn attribute has a method named attenuate + opt_counter=self.nn.opt_counter[0] # assign the first element of the opt_counter attribute of the nn attribute to the opt_counter variable + opt_counter+=1 # increment each element of the opt_counter variable by 1 + self.nn.opt_counter[0]=opt_counter # assign the opt_counter variable to the first element of the opt_counter attribute of the nn attribute + if hasattr(self.nn,'bc'): # check if the nn attribute has an attribute named bc + bc=self.nn.bc[0] # assign the first element of the bc attribute of the nn attribute to the bc variable + bc+=1 # increment the bc variable by 1 + self.nn.bc[0]=bc # assign the bc variable to the first element of the bc attribute of the nn attribute + try: # try to execute the following code block + if hasattr(self.nn,'accuracy'): # check if the nn attribute has a method named accuracy + try: # try to execute the following code block + batch_acc=self.nn.accuracy(output,labels_batch,p) # call the accuracy method of the nn attribute with the output, labels_batch, and p arguments, and assign the return value to the batch_acc variable + except Exception: # handle any exception that may occur + batch_acc=self.nn.accuracy(output,labels_batch) # call the accuracy method of the nn attribute with the output and labels_batch arguments, and assign the return value to the batch_acc variable + except Exception as e: # handle any exception that may occur + raise e # raise the exception again + if hasattr(self.nn,'accuracy'): # check if the nn attribute has a method named accuracy + self.total_loss[p]+=batch_loss # increment the element of the total_loss attribute at the index of p by the batch_loss variable + self.total_acc[p]+=batch_acc # increment the element of the total_acc attribute at the index of p by the batch_acc variable + else: # otherwise + self.total_loss[p]+=batch_loss # increment the element of the total_loss attribute at the index of p by the batch_loss variable + self.batch_counter[p]+=1 # increment the element of the batch_counter attribute at the index of p by 1 + if lock is not None: # check if lock argument is not None + lock.acquire() # call the acquire method of lock argument to acquire a lock for synchronization + batches=np.sum(self.batch_counter) # compute the sum of all elements in the batch_counter attribute and assign it to batches variable + if batches>=len(train_loader): # check if batches is greater than or equal to length of train_loader argument + batch_counter=np.frombuffer(self.batch_counter.get_obj(),dtype='i') # create a numpy array from a buffer object that contains a copy of memory from another object, using an integer data type, and assign it to batch_counter variable + batch_counter*=0 # multiply each element of batch_counter variable by 0 + loss=np.sum(self.total_loss)/batches # compute the average of all elements in total_loss attribute and assign it to loss variable + if hasattr(self.nn,'accuracy'): # check if nn attribute has a method named accuracy + train_acc=np.sum(self.total_acc)/batches # compute the average of all elements in total_acc attribute and assign it to train_acc variable + self.total_epoch.value+=1 # increment value of total_epoch attribute by 1 + self.train_loss.value=loss # assign loss variable to value of train_loss attribute + self.train_loss_list.append(loss) # append loss variable to train_loss_list attribute + if hasattr(self.nn,'accuracy'): # check if nn attribute has a method named accuracy + self.train_acc.value=train_acc # assign train_acc variable to value of train_acc attribute + self.train_acc_list.append(train_acc) # append train_acc variable to train_acc_list attribute + if self.test_flag==True: # check if test_flag attribute is True + if hasattr(self.nn,'accuracy'): # check if nn attribute has a method named accuracy + self.test_loss.value,self.test_acc.value=self.test(test_batch) # call the test method with the test_batch argument, and assign the return values to the value of the test_loss attribute and the value of the test_acc attribute + self.test_loss_list.append(self.test_loss.value) # append the value of the test_loss attribute to the test_loss_list attribute + self.test_acc_list.append(self.test_acc.value) # append the value of the test_acc attribute to the test_acc_list attribute + else: # otherwise + self.test_loss.value=self.test(test_batch) # call the test method with the test_batch argument, and assign the return value to the value of the test_loss attribute + self.test_loss_list.append(self.test_loss.value) # append the value of the test_loss attribute to the test_loss_list attribute + self.print_save() # call the print_save method to print and save some information + self.epoch_counter.value+=1 # increment the value of the epoch_counter attribute by 1 + if hasattr(self.nn,'ec'): # check if the nn attribute has an attribute named ec + ec=self.nn.ec[0] # assign the first element of the ec attribute of the nn attribute to the ec variable + ec+=1 # increment the ec variable by 1 + self.nn.ec[0]=ec # assign the ec variable to the first element of the ec attribute of the nn attribute + total_loss=np.frombuffer(self.total_loss.get_obj(),dtype='f') # create a numpy array from a buffer object that contains a copy of memory from another object, using a float data type, and assign it to total_loss variable + total_loss*=0 # multiply each element of total_loss variable by 0 + if hasattr(self.nn,'accuracy'): # check if nn attribute has a method named accuracy + total_acc=np.frombuffer(self.total_acc.get_obj(),dtype='f') # create a numpy array from a buffer object that contains a copy of memory from another object, using a float data type, and assign it to total_acc variable + total_acc*=0 # multiply each element of total_acc variable by 0 + if lock is not None: # check if lock argument is not None + lock.release() # call the release method of lock argument to release a lock for synchronization + if self.epoch_counter.value>=self.epoch: # check if value of epoch_counter attribute is greater than or equal to epoch attribute + return # return nothing + + + def train(self,p,lock=None,test_batch=None): + '''define a method named train that takes three arguments: p, lock, and test_batch''' + + '''This method is used to train the neural network for one epoch using a given train_dataset, + using the p argument as the index of the process, + using the lock argument as a multiprocessing.Lock object for synchronization, + and using the test_batch argument as a batch of test data for evaluation''' + + '''The p argument is expected to be an integer that represents the index of the process that calls this method''' + + '''The lock argument is expected to be a multiprocessing.Lock object that can be used to acquire and release a lock for synchronization across processes''' + + '''The test_batch argument is expected to be a PyTorch tensor that represents a batch of test data for evaluation''' + if type(self.train_dataset)==list: # check if the train_dataset attribute is a list + train_loader=torch.utils.data.DataLoader(self.train_dataset[p],batch_size=self.batch) # create a PyTorch DataLoader object from the element of the train_dataset attribute at the index of p, using the batch attribute as the batch size, and assign it to the train_loader variable + else: # otherwise + train_loader=torch.utils.data.DataLoader(self.train_dataset,batch_size=self.batch,shuffle=self.shuffle) # create a PyTorch DataLoader object from the train_dataset attribute, using the batch attribute as the batch size and the shuffle attribute as the shuffle flag, and assign it to the train_loader variable + self.train7(train_loader,p,test_batch,lock) # call the train7 method with the train_loader, p, test_batch, and lock arguments + return # return nothing + + + def train_online(self,p,lock=None): + '''define a method named train_online that takes two arguments: p and lock''' + + '''This method is used to train the neural network online using a given online method of the neural network, + using the p argument as the index of the process, + and using the lock argument as a multiprocessing.Lock object for synchronization''' + + '''The p argument is expected to be an integer that represents the index of the process that calls this method''' + + '''The lock argument is expected to be a multiprocessing.Lock object that can be used to acquire and release a lock for synchronization across processes''' + + if hasattr(self.nn,'counter'): # check if the nn attribute has an attribute named counter + self.nn.counter.append(0) # append 0 to the counter attribute of the nn attribute + while True: # enter an infinite loop + if hasattr(self.nn,'save'): # check if the nn attribute has a method named save + self.nn.save(self.save,p) # call the save method of the nn attribute with the save and p arguments + if hasattr(self.nn,'stop_flag'): # check if the nn attribute has an attribute named stop_flag + if self.nn.stop_flag==True: # check if the value of the stop_flag attribute of the nn attribute is True + return # return nothing + if hasattr(self.nn,'stop_func'): # check if the nn attribute has a method named stop_func + if self.nn.stop_func(p): # call the stop_func method of the nn attribute with the p argument, and check if it returns True + return # return nothing + if hasattr(self.nn,'suspend_func'): # check if the nn attribute has a method named suspend_func + self.nn.suspend_func(p) # call the suspend_func method of the nn attribute with the p argument + try: # try to execute the following code block + data=self.nn.online(p) # call the online method of the nn attribute with the p argument, and assign the return value to data variable + except Exception as e: # handle any exception that may occur + self.nn.exception_list[p]=e # assign e to the element of the exception_list attribute of the nn attribute at the index of p + if data=='stop': # check if data is equal to 'stop' + return # return nothing + elif data=='suspend': # check if data is equal to 'suspend' + self.nn.suspend_func(p) # call the suspend_func method of the nn attribute with the p argument + try: # try to execute the following code block + output,loss,param=self.opt(data[0],data[1],p,lock) # call the opt method with the first and second elements of data, p, and lock arguments, and assign the return values to output, loss, and param variables + self.param[7]=param # assign param variable to the seventh element of the param attribute + except Exception as e: # handle any exception that may occur + if lock!=None: # check if lock argument is not None + if lock.acquire(False): # call the acquire method of lock argument with False argument, and check if it returns True + lock.release() # call the release method of lock argument to release a lock for synchronization + self.nn.exception_list[p]=e # assign e to the element of the exception_list attribute of the nn attribute at the index of p + loss=loss.numpy() # convert loss variable to a numpy array and assign it to loss variable + if len(self.nn.train_loss_list)==self.nn.max_length: # check if length of train_loss_list attribute of nn attribute is equal to max_length attribute of nn attribute + del self.nn.train_loss_list[0] # delete the first element of train_loss_list attribute of nn attribute + self.nn.train_loss_list.append(loss) # append loss variable to train_loss_list attribute of nn attribute + try: # try to execute the following code block + if hasattr(self.nn,'accuracy'): # check if nn attribute has a method named accuracy + try: # try to execute the following code block + acc=self.nn.accuracy(output,data[1]) # call the accuracy method of nn attribute with output and second element of data arguments, and assign the return value to acc variable + except Exception: # handle any exception that may occur + self.exception_list[p]=True # assign True to the element of exception_list attribute at the index of p + if len(self.nn.train_acc_list)==self.nn.max_length: # check if length of train_acc_list attribute of nn attribute is equal to max_length attribute of nn attribute + del self.nn.train_acc_list[0] # delete the first element of train_acc_list attribute of nn attribute + self.nn.train_acc_list.append(acc) # append acc variable to train_acc_list attribute of nn attribute + except Exception as e: # handle any exception that may occur + self.nn.exception_list[p]=e # assign e to the element of exception_list attribute of nn attribute at the index of p + try: # try to execute the following code block + if hasattr(self.nn,'counter'): # check if nn attribute has an attribute named counter + count=self.nn.counter[p] # assign the element of counter attribute of nn attribute at the index of p to count variable + count+=1 # increment count variable by 1 + self.nn.counter[p]=count # assign count variable to the element of counter attribute of nn attribute at the index of p + except IndexError: # handle any IndexError that may occur + self.nn.counter.append(0) # append 0 to counter attribute of nn attribute + count=self.nn.counter[p] # assign the element of counter attribute of nn attribute at the index of p to count variable + count+=1 # increment count variable by 1 + self.nn.counter[p]=count # assign count variable to the element of counter attribute of nn attribute at the index of p + return # return nothing + + + def test_(self,data,labels): + '''define a method named test_ that takes two arguments: data and labels''' + + '''This method is used to compute the test loss and test accuracy for a given batch of test data, + using the data and labels arguments as the input and output of the neural network''' + + '''The data and labels arguments are expected to be PyTorch tensors that represent a batch of input and output data for the neural network''' + + try: # try to execute the following code block + try: # try to execute the following code block + output=self.nn.fp(data) # call the fp method of nn attribute with data argument, and assign the return value to output variable + loss=self.nn.loss(output,labels) # call the loss method of nn attribute with output and labels arguments, and assign the return value to loss variable + except Exception: # handle any exception that may occur + output,loss=self.nn.fp(data,labels) # call the fp method of nn attribute with data and labels arguments, and assign the return values to output and loss variables + except Exception as e: # handle any exception that may occur + raise e # raise the exception again + try: # try to execute the following code block + acc=self.nn.accuracy(output,labels) # call the accuracy method of nn attribute with output and labels arguments, and assign the return value to acc variable + except Exception as e: # handle any exception that may occur + if hasattr(self.nn,'accuracy'): # check if nn attribute has a method named accuracy + raise e # raise the exception again + else: # otherwise + acc=None # assign None to acc variable + return loss,acc # return loss and acc variables + + + def test(self,batch=None): + '''define a method named test that takes one argument: batch''' + + '''This method is used to compute the test loss and test accuracy for the whole test dataset or a given batch of test data, + using the batch argument as a batch of test data for evaluation''' + + '''The batch argument is expected to be a PyTorch tensor that represents a batch of test data for evaluation, or None to use the whole test dataset''' + total_loss=0 # initialize total_loss variable to 0 + total_acc=0 # initialize total_acc variable to 0 + batches=0 # initialize batches variable to 0 + if batch is None: # check if batch argument is None + test_loader=torch.utils.data.DataLoader(self.test_dataset,batch_size=self.batch) # create a PyTorch DataLoader object from the test_dataset attribute, using the batch attribute as the batch size, and assign it to the test_loader variable + else: # otherwise + test_loader=[batch] # create a list containing the batch argument and assign it to the test_loader variable + for data_batch,labels_batch in test_loader: # iterate over the batches of test data from the test_loader + batches+=1 # increment batches variable by 1 + batch_loss,batch_acc=self.test_(data_batch,labels_batch) # call the test_ method with the data_batch and labels_batch arguments, and assign the return values to batch_loss and batch_acc variables + total_loss+=batch_loss # increment total_loss variable by batch_loss variable + if hasattr(self.nn,'accuracy'): # check if nn attribute has a method named accuracy + total_acc+=batch_acc # increment total_acc variable by batch_acc variable + test_loss=total_loss.detach().numpy()/batches # compute the average of total_loss variable and convert it to a numpy array and assign it to test_loss variable + if hasattr(self.nn,'accuracy'): # check if nn attribute has a method named accuracy + test_acc=total_acc.detach().numpy()/batches # compute the average of total_acc variable and convert it to a numpy array and assign it to test_acc variable + if hasattr(self.nn,'accuracy'): # check if nn attribute has a method named accuracy + return test_loss,test_acc # return test_loss and test_acc variables + else: # otherwise + return test_loss # return test_loss variable \ No newline at end of file From 4e126b74233815da4ee7ccf1485524f30d4f447d Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Sun, 23 Jul 2023 10:49:16 +0800 Subject: [PATCH 077/337] Update kernel.py --- Note 7.0 documentation/RL/kernel/kernel.py | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/Note 7.0 documentation/RL/kernel/kernel.py b/Note 7.0 documentation/RL/kernel/kernel.py index 65ef15d1..664910c8 100644 --- a/Note 7.0 documentation/RL/kernel/kernel.py +++ b/Note 7.0 documentation/RL/kernel/kernel.py @@ -139,16 +139,9 @@ def pool(self,s,a,next_s,r,done,pool_lock,index): if type(self.state_pool[index])!=np.ndarray and self.state_pool[index]==None: # check if the pool is empty self.state_pool[index]=s # assign the state to the pool if type(a)==int: # check if the action is an integer - if type(self.nn.param[0])!=list: - a=np.array(a,self.nn.param[0].dtype.name) # convert the action to an array with the same data type as neural network parameters - else: - a=np.array(a,self.nn.param[0][0].dtype.name) # convert the action to an array with the same data type as neural network parameters + a=np.array(a) # convert the action to an array self.action_pool[index]=np.expand_dims(a,axis=0) # expand the dimension of the action and assign it to the pool else: # otherwise, assume the action is an array - if type(self.nn.param[0])!=list: - a=a.astype(self.nn.param[0].dtype.name) # convert the action to an array with the same data type as neural network parameters - else: - a=a.astype(self.nn.param[0][0].dtype.name) # convert the action to an array with the same data type as neural network parameters self.action_pool[index]=a # assign the action to the pool self.next_state_pool[index]=np.expand_dims(next_s,axis=0) # expand the dimension of the next state and assign it to the pool self.reward_pool[index]=np.expand_dims(r,axis=0) # expand the dimension of the reward and assign it to the pool @@ -157,16 +150,9 @@ def pool(self,s,a,next_s,r,done,pool_lock,index): try: self.state_pool[index]=np.concatenate((self.state_pool[index],s),0) # concatenate the state with the existing pool if type(a)==int: # check if the action is an integer - if type(self.nn.param[0])!=list: - a=np.array(a,self.nn.param[0].dtype.name) # convert the action to an array with the same data type as neural network parameters - else: - a=np.array(a,self.nn.param[0][0].dtype.name) # convert the action to an array with the same data type as neural network parameters + a=np.array(a) # convert the action to an array self.action_pool[index]=np.concatenate((self.action_pool[index],np.expand_dims(a,axis=0)),0) # expand and concatenate the action with the existing pool else: # otherwise, assume the action is an array - if type(self.nn.param[0])!=list: - a=a.astype(self.nn.param[0].dtype.name) # convert the action to an array with the same data type as neural network parameters - else: - a=a.astype(self.nn.param[0][0].dtype.name) # convert the action to an array with the same data type as neural network parameters self.action_pool[index]=np.concatenate((self.action_pool[index],a),0) # concatenate the action with the existing pool self.next_state_pool[index]=np.concatenate((self.next_state_pool[index],np.expand_dims(next_s,axis=0)),0) # expand and concatenate the next state with the existing pool self.reward_pool[index]=np.concatenate((self.reward_pool[index],np.expand_dims(r,axis=0)),0) # expand and concatenate the reward with the existing pool From 1dbf0658d365c56465662fb2ddc8206f4a9ffafe Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Sun, 23 Jul 2023 11:07:15 +0800 Subject: [PATCH 078/337] Delete kernel.py --- .../RL/kernel/nspn/kernel.py | 1173 ----------------- 1 file changed, 1173 deletions(-) delete mode 100644 Note 7.0 documentation/RL/kernel/nspn/kernel.py diff --git a/Note 7.0 documentation/RL/kernel/nspn/kernel.py b/Note 7.0 documentation/RL/kernel/nspn/kernel.py deleted file mode 100644 index a6ee7dc5..00000000 --- a/Note 7.0 documentation/RL/kernel/nspn/kernel.py +++ /dev/null @@ -1,1173 +0,0 @@ -# import the necessary modules -from tensorflow import function -from tensorflow import data as tf_data -import numpy as np -import statistics -import time - -# define a class named kernel -class kernel: - # define the initialization method - def __init__(self,nn=None,save_episode=False): - # assign the nn parameter to the self.nn attribute - self.nn=nn - # if the nn attribute has a km attribute, assign 1 to it - if hasattr(self.nn,'km'): - self.nn.km=1 - # initialize the platform attribute as None - self.platform=None - # initialize the state_pool, action_pool, next_state_pool, reward_pool, done_pool attributes as None - self.state_pool=None - self.action_pool=None - self.next_state_pool=None - self.reward_pool=None - self.done_pool=None - # initialize the episode_set attribute as an empty list - self.episode_set=[] - # initialize the epsilon attribute as None - self.epsilon=None - # initialize the episode_step attribute as None - self.episode_step=None - # initialize the pool_size attribute as None - self.pool_size=None - # initialize the batch attribute as None - self.batch=None - # initialize the update_step attribute as None - self.update_step=None - # initialize the trial_count attribute as None - self.trial_count=None - # initialize the criterion attribute as None - self.criterion=None - # initialize the reward_list attribute as an empty list - self.reward_list=[] - # initialize the suspend attribute as False - self.suspend=False - # initialize the save_epi attribute as None - self.save_epi=None - # initialize the max_episode_count attribute as None - self.max_episode_count=None - # assign the save_episode parameter to the save_episode attribute - self.save_episode=save_episode - # assign the filename parameter to the filename attribute - self.filename='save.dat' - # initialize the loss attribute as 0 - self.loss=0 - # initialize the loss_list attribute as an empty list - self.loss_list=[] - # initialize the sc attribute as 0 - self.sc=0 - # initialize the total_episode attribute as 0 - self.total_episode=0 - # initialize the time attribute as 0 - self.time=0 - # initialize the total_time attribute as 0 - self.total_time=0 - - # define a method named action_vec - def action_vec(self): - # if the epsilon attribute is not None, assign a numpy array of ones with the length of action_count and multiply it by epsilon divided by the length of action_one to the action_one attribute - if self.epsilon!=None: - self.action_one=np.ones(self.action_count,dtype=np.int8) - # return nothing - return - - # define a method named init - def init(self): - # try to execute the following statements - try: - # if the nn attribute has a pr attribute, assign a numpy array of 0 to its TD attribute - if hasattr(self.nn,'pr'): - self.nn.pr.TD=np.array(0) - # if an exception occurs, raise it - except Exception as e: - raise e - # assign False to the suspend attribute - self.suspend=False - # assign None to the save_epi attribute - self.save_epi=None - # assign an empty list to the episode_set attribute - self.episode_set=[] - # assign None to the state_pool, action_pool, next_state_pool, reward_pool, done_pool attributes - self.state_pool=None - self.action_pool=None - self.next_state_pool=None - self.reward_pool=None - self.done_pool=None - # assign an empty list to the reward_list attribute - self.reward_list=[] - # assign 0 to the loss attribute - self.loss=0 - # assign an empty list to the loss_list attribute - self.loss_list=[] - # assign 0 to the sc attribute - self.sc=0 - # assign 0 to the total_episode attribute - self.total_episode=0 - # assign 0 to the time attribute - self.time=0 - # assign 0 to the total_time attribute - self.total_time=0 - - # define a method named set_up - def set_up(self,epsilon=None,episode_step=None,pool_size=None,batch=None,update_step=None,trial_count=None,criterion=None): - # if epsilon is not None, assign it to the epsilon attribute - if epsilon!=None: - self.epsilon=epsilon - # if episode_step is not None, assign it to the episode_step attribute - if episode_step!=None: - self.episode_step=episode_step - # if pool_size is not None, assign it to the pool_size attribute - if pool_size!=None: - self.pool_size=pool_size - # if batch is not None, assign it to the batch attribute - if batch!=None: - self.batch=batch - # if update_step is not None, assign it to the update_step attribute - if update_step!=None: - self.update_step=update_step - # if trial_count is not None, assign it to the trial_count attribute - if trial_count!=None: - self.trial_count=trial_count - # if criterion is not None, assign it to the criterion attribute - if criterion!=None: - self.criterion=criterion - # call the action_vec method - self.action_vec() - # return nothing - return - - # define a method named epsilon_greedy_policy - def epsilon_greedy_policy(self,s): - # assign a numpy array of ones with the length of action_count and multiply it by epsilon divided by the length of action_one to the action_prob variable - action_prob=self.action_one*self.epsilon/len(self.action_one) - # try to execute the following statements - try: - # assign the index of the maximum value of the output of the nn attribute's fp method with s as input to the best_a variable - best_a=np.argmax(self.nn.nn.fp(s)) - # add 1 minus epsilon to the best_a element of action_prob - action_prob[best_a]+=1-self.epsilon - # if an exception occurs, execute the following statements - except Exception as e: - # if the platform attribute has a DType attribute, raise the exception - if hasattr(self.platform,'DType'): - raise e - # else, assign the index of the maximum value of the output of the nn attribute's nn method with s as input to the best_a variable - else: - best_a=self.nn.nn(s).argmax() - # add 1 minus epsilon to the best_a element of action_prob and convert it to numpy array - action_prob[best_a.numpy()]+=1-self.epsilon - # return action_prob - return action_prob - - # define a method named get_reward - def get_reward(self,max_step=None,seed=None): - # initialize the reward variable as 0 - reward=0 - # if seed is None, assign the output of the genv attribute's reset method to the s variable - if seed==None: - s=self.genv.reset() - # else, assign the output of the genv attribute's reset method with seed as input to the s variable - else: - s=self.genv.reset(seed=seed) - # if max_step is not None, execute the following statements - if max_step!=None: - # use a for loop to iterate from 0 to max_step - for i in range(max_step): - # if the end_flag attribute is True, break the loop - if self.end_flag==True: - break - # if the nn attribute has a nn attribute, execute the following statements - if hasattr(self.nn,'nn'): - # if the platform attribute has a DType attribute, execute the following statements - if hasattr(self.platform,'DType'): - # expand the dimension of s along axis 0 and assign it back to s - s=np.expand_dims(s,axis=0) - # assign the index of the maximum value of the output of the nn attribute's nn method with s as input to the a variable - a=np.argmax(self.nn.nn.fp(s)) - # else, execute the following statements - else: - # expand the dimension of s along axis 0 and assign it back to s - s=np.expand_dims(s,axis=0) - # convert s to a tensor with float dtype and assign it to the device attribute of the nn attribute and assign it back to s - s=self.platform.tensor(s,dtype=self.platform.float).to(self.nn.device) - # assign the index of the maximum value of the output of the nn attribute's nn method with s as input to the a variable and convert it to numpy array - a=self.nn.nn(s).detach().numpy().argmax() - # else, execute the following statements - else: - # if the nn attribute has an action attribute, execute the following statements - if hasattr(self.nn,'action'): - # if the platform attribute has a DType attribute, execute the following statements - if hasattr(self.platform,'DType'): - # expand the dimension of s along axis 0 and assign it back to s - s=np.expand_dims(s,axis=0) - # assign the output of the nn attribute's action method with s as input to the a variable and convert it to numpy array - a=self.nn.action(s).numpy() - # else, execute the following statements - else: - # expand the dimension of s along axis 0 and assign it back to s - s=np.expand_dims(s,axis=0) - # convert s to a tensor with float dtype and assign it to the device attribute of the nn attribute and assign it back to s - s=self.platform.tensor(s,dtype=self.platform.float).to(self.nn.device) - # assign the output of the nn attribute's action method with s as input to the a variable and convert it to numpy array and detach it from computation graph - a=self.nn.action(s).detach().numpy() - # else, execute the following statements - else: - # if the platform attribute has a DType attribute, execute the following statements - if hasattr(self.platform,'DType'): - # expand the dimension of s along axis 0 and assign it back to s - s=np.expand_dims(s,axis=0) - # assign the output of adding up output of fp method from actor attribute of nn with noise from nn as input to a variable and convert it to numpy array - a=(self.nn.actor.fp(s)+self.nn.noise()).numpy() - # squeeze out any singleton dimensions from a and assign it back to a - a=np.squeeze(a) - # else, execute the following statements - else: - # expand the dimension of s along axis 0 and assign it back to s - s=np.expand_dims(s,axis=0) - # convert s to a tensor with float dtype and assign it to device attribute of nn and assign it back to s - s=self.platform.tensor(s,dtype=self.platform.float).to(self.nn.device) - # assign output of adding up output from actor method from nn with noise from nn as input to a variable and convert it to numpy array and detach it from computation graph - a=(self.nn.actor(s)+self.nn.noise()).detach().numpy() - # squeeze out any singleton dimensions from a and assign it back to a - a=np.squeeze(a) - # assign the output of the genv attribute's step method with a as input to the next_s, r, done variables - next_s,r,done,_=self.genv.step(a) - # assign next_s to s - s=next_s - # add r to reward and assign it back to reward - reward+=r - # if the nn attribute has a stop attribute, execute the following statements - if hasattr(self.nn,'stop'): - # if the output of the nn attribute's stop method with next_s as input is True, break the loop - if self.nn.stop(next_s): - break - # if done is True, break the loop - if done: - break - # return reward - return reward - # else, execute the following statements - else: - # use a while loop to iterate indefinitely - while True: - # if the end_flag attribute is True, break the loop - if self.end_flag==True: - break - # if the nn attribute has a nn attribute, execute the following statements - if hasattr(self.nn,'nn'): - # if the platform attribute has a DType attribute, execute the following statements - if hasattr(self.platform,'DType'): - # expand the dimension of s along axis 0 and assign it back to s - s=np.expand_dims(s,axis=0) - # assign the index of the maximum value of the output of the nn attribute's nn method with s as input to the a variable - a=np.argmax(self.nn.nn.fp(s)) - # else, execute the following statements - else: - # expand the dimension of s along axis 0 and assign it back to s - s=np.expand_dims(s,axis=0) - # convert s to a tensor with float dtype and assign it to the device attribute of the nn attribute and assign it back to s - s=self.platform.tensor(s,dtype=self.platform.float).to(self.nn.device) - # assign the index of the maximum value of the output of the nn attribute's nn method with s as input to the a variable and convert it to numpy array - a=self.nn.nn(s).detach().numpy().argmax() - # else, execute the following statements - else: - # if the nn attribute has an action attribute, execute the following statements - if hasattr(self.nn,'action'): - # if the platform attribute has a DType attribute, execute the following statements - if hasattr(self.platform,'DType'): - # expand the dimension of s along axis 0 and assign it back to s - s=np.expand_dims(s,axis=0) - # assign the output of the nn attribute's action method with s as input to the a variable and convert it to numpy array - a=self.nn.action(s).numpy() - # else, execute the following statements - else: - # expand the dimension of s along axis 0 and assign it back to s - s=np.expand_dims(s,axis=0) - # convert s to a tensor with float dtype and assign it to device attribute of nn and assign it back to s - s=self.platform.tensor(s,dtype=self.platform.float).to(self.nn.device) - # assign output of action method from nn with s as input to a variable and convert it to numpy array and detach it from computation graph - a=self.nn.action(s).detach().numpy() - # else, execute the following statements - else: - # if platform has DType attribute, execute following statements - if hasattr(self.platform,'DType'): - # expand dimension of s along axis 0 and assign it back to s - s=np.expand_dims(s,axis=0) - # assign output of adding up output from fp method from actor attribute of nn with noise from nn as input to a variable and convert it to numpy array - a=(self.nn.actor.fp(s)+self.nn.noise()).numpy() - # squeeze out any singleton dimensions from a and assign it back to a - a=np.squeeze(a) - # else, execute following statements - else: - # expand dimension of s along axis 0 and assign it back to s - s=np.expand_dims(s,axis=0) - # convert s to tensor with float dtype and assign it to device attribute of nn and assign it back to s - s=self.platform.tensor(s,dtype=self.platform.float).to(self.nn.device) - # assign output of adding up output from actor method from nn with noise from nn as input to a variable and convert it to numpy array and detach it from computation graph - a=(self.nn.actor(s)+self.nn.noise()).detach().numpy() - # squeeze out any singleton dimensions from a and assign it back to a - a=np.squeeze(a) - # assign output of genv attribute's step method with a as input to next_s, r, done variables - next_s,r,done,_=self.genv.step(a) - # assign next_s to s - s=next_s - # add r to reward and assign it back to reward - reward+=r - # if the nn attribute has a stop attribute, execute the following statements - if hasattr(self.nn,'stop'): - # if the output of the nn attribute's stop method with next_s as input is True, break the loop - if self.nn.stop(next_s): - break - # if done is True, break the loop - if done: - break - # return reward - return reward - - # define a method named get_episode - def get_episode(self,max_step=None,seed=None): - # initialize the counter variable as 0 - counter=0 - # initialize the episode variable as an empty list - episode=[] - # if seed is None, assign the output of the genv attribute's reset method to the s variable - if seed==None: - s=self.genv.reset() - # else, assign the output of the genv attribute's reset method with seed as input to the s variable - else: - s=self.genv.reset(seed=seed) - # initialize the end_flag attribute as False - self.end_flag=False - # use a while loop to iterate indefinitely - while True: - # if the end_flag attribute is True, break the loop - if self.end_flag==True: - break - # if the nn attribute has a nn attribute, execute the following statements - if hasattr(self.nn,'nn'): - # if the platform attribute has a DType attribute, execute the following statements - if hasattr(self.platform,'DType'): - # expand the dimension of s along axis 0 and assign it back to s - s=np.expand_dims(s,axis=0) - # assign the index of the maximum value of the output of the nn attribute's nn method with s as input to the a variable - a=np.argmax(self.nn.nn.fp(s)) - # else, execute the following statements - else: - # expand the dimension of s along axis 0 and assign it back to s - s=np.expand_dims(s,axis=0) - # convert s to a tensor with float dtype and assign it to the device attribute of the nn attribute and assign it back to s - s=self.platform.tensor(s,dtype=self.platform.float).to(self.nn.device) - # assign the index of the maximum value of the output of the nn attribute's nn method with s as input to the a variable and convert it to numpy array - a=self.nn.nn(s).detach().numpy().argmax() - # else, execute the following statements - else: - # if the nn attribute has an action attribute, execute the following statements - if hasattr(self.nn,'action'): - # if the platform attribute has a DType attribute, execute the following statements - if hasattr(self.platform,'DType'): - # expand the dimension of s along axis 0 and assign it back to s - s=np.expand_dims(s,axis=0) - # assign output of action method from nn with s as input to a variable and convert it to numpy array - a=self.nn.action(s).numpy() - # else, execute following statements - else: - # expand dimension of s along axis 0 and assign it back to s - s=np.expand_dims(s,axis=0) - # convert s to tensor with float dtype and assign it to device attribute of nn and assign it back to s - s=self.platform.tensor(s,dtype=self.platform.float).to(self.nn.device) - # assign output of action method from nn with s as input to a variable and convert it to numpy array and detach it from computation graph - a=self.nn.action(s).detach().numpy() - # else, execute following statements - else: - # if platform has DType attribute, execute following statements - if hasattr(self.platform,'DType'): - # expand dimension of s along axis 0 and assign it back to s - s=np.expand_dims(s,axis=0) - # assign output of adding up output from fp method from actor attribute of nn with noise from nn as input to a variable and convert it to numpy array - a=(self.nn.actor.fp(s)+self.nn.noise()).numpy() - # squeeze out any singleton dimensions from a and assign it back to a - a=np.squeeze(a) - # else, execute following statements - else: - # expand dimension of s along axis 0 and assign it back to s - s=np.expand_dims(s,axis=0) - # convert s to tensor with float dtype and assign it to device attribute of nn and assign it back to s - s=self.platform.tensor(s,dtype=self.platform.float).to(self.nn.device) - # assign output of adding up output from actor method from nn with noise from nn as input to a variable and convert it to numpy array and detach it from computation graph - a=(self.nn.actor(s)+self.nn.noise()).detach().numpy() - # squeeze out any singleton dimensions from a and assign it back to a - a=np.squeeze(a) - # assign the output of the genv attribute's step method with a as input to the next_s, r, done variables - next_s,r,done,_=self.genv.step(a) - # if the end_flag attribute is True, break the loop - if self.end_flag==True: - break - # if done is True, execute the following statements - if done: - # append a list of s, a, next_s, r to the episode variable - episode.append([s,a,next_s,r]) - # append the string 'done' to the episode variable - episode.append('done') - # break the loop - break - # else, execute the following statements - else: - # append a list of s, a, next_s, r to the episode variable - episode.append([s,a,next_s,r]) - # if max_step is not None and counter equals max_step minus 1, break the loop - if max_step!=None and counter==max_step-1: - break - # assign next_s to s - s=next_s - # increment counter by 1 - counter+=1 - # return episode - return episode - - # define a method named tf_opt - @function(jit_compile=True) - def tf_opt(self,state_batch,action_batch,next_state_batch,reward_batch,done_batch): - # use the platform attribute's GradientTape method with persistent as True as a context manager and assign it to the tape variable - with self.platform.GradientTape(persistent=True) as tape: - # assign the output of the nn attribute's loss method with state_batch, action_batch, next_state_batch, reward_batch, done_batch as inputs to the loss variable - loss=self.nn.loss(state_batch,action_batch,next_state_batch,reward_batch,done_batch) - # if the nn attribute has a gradient attribute, execute the following statements - if hasattr(self.nn,'gradient'): - # assign the output of the nn attribute's gradient method with tape and loss as inputs to the gradient variable - gradient=self.nn.gradient(tape,loss) - # if the opt attribute of the nn attribute has an apply_gradients method, execute the following statements - if hasattr(self.nn.opt,'apply_gradients'): - # call the apply_gradients method of the opt attribute of the nn attribute with zip function of gradient and param attribute of nn as input - self.nn.opt.apply_gradients(zip(gradient,self.nn.param)) - # else, execute following statements - else: - # call opt attribute of nn with gradient as input - self.nn.opt(gradient) - # else, execute following statements - else: - # if nn has nn attribute, execute following statements - if hasattr(self.nn,'nn'): - # assign output of tape's gradient method with loss and param attribute of nn as inputs to gradient variable - gradient=tape.gradient(loss,self.nn.param) - # call apply_gradients method of opt attribute of nn with zip function of gradient and param attribute of nn as input - self.nn.opt.apply_gradients(zip(gradient,self.nn.param)) - # else, execute following statements - else: - # assign output of tape's gradient method with first element of loss and first element of param attribute of nn as inputs to actor_gradient variable - actor_gradient=tape.gradient(loss[0],self.nn.param[0]) - # assign output of tape's gradient method with second element of loss and second element of param attribute of nn as inputs to critic_gradient variable - critic_gradient=tape.gradient(loss[1],self.nn.param[1]) - # call apply_gradients method of opt attribute of nn with zip function of actor_gradient and first element of param attribute of nn as input - self.nn.opt.apply_gradients(zip(actor_gradient,self.nn.param[0])) - # call apply_gradients method of opt attribute of nn with zip function of critic_gradient and second element of param attribute of nn as input - self.nn.opt.apply_gradients(zip(critic_gradient,self.nn.param[1])) - # return loss - return loss - - # define a method named pytorch_opt - def pytorch_opt(self,state_batch,action_batch,next_state_batch,reward_batch,done_batch): - # assign output of loss method from nn with state_batch, action_batch, next_state_batch, reward_batch, done_batch as inputs to loss variable - loss=self.nn.loss(state_batch,action_batch,next_state_batch,reward_batch,done_batch) - # assign next_state_batch to loss variable - loss=self.nn.loss(state_batch,action_batch,next_state_batch,reward_batch,done_batch) - # call zero_grad method of opt attribute of nn - self.nn.opt.zero_grad() - # if nn has nn attribute, execute following statements - if hasattr(self.nn,'nn'): - # call backward method of loss - loss.backward() - # call step method of opt attribute of nn - self.nn.opt.step() - # else, execute following statements - else: - # call backward method of first element of loss - loss[0].backward() - # call step method of opt attribute of nn - self.nn.opt.step() - # call zero_grad method of opt attribute of nn - self.nn.opt.zero_grad() - # call backward method of second element of loss - loss[1].backward() - # call step method of opt attribute of nn - self.nn.opt.step() - # return loss - return loss - - # define a method named opt - def opt(self,state_batch,action_batch,next_state_batch,reward_batch,done_batch): - # if the platform attribute has a DType attribute, execute the following statements - if hasattr(self.platform,'DType'): - # call the tf_opt method with state_batch, action_batch, next_state_batch, reward_batch, done_batch as inputs and assign the output to the loss variable - loss=self.tf_opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch) - # else, execute the following statements - else: - # call the pytorch_opt method with state_batch, action_batch, next_state_batch, reward_batch, done_batch as inputs and assign the output to the loss variable - loss=self.pytorch_opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch) - # return loss - return loss - - # define a method named opt_ol - def opt_ol(self,state,action,next_state,reward,done): - # if the platform attribute has a DType attribute, execute the following statements - if hasattr(self.platform,'DType'): - # call the tf_opt method with state, action, next_state, reward, done as inputs and assign the output to loss variable - loss=self.tf_opt(state,action,next_state,reward,done) - # else, execute the following statements - else: - # call the pytorch_opt method with state, action, next_state, reward, done as inputs and assign the output to loss variable - loss=self.pytorch_opt(state,action,next_state,reward,done) - # return the loss variable - return loss - - # define a method named pool - def pool(self,s,a,next_s,r,done): - # if the state_pool attribute is None, execute the following statements - if self.state_pool==None: - # expand the dimension of s along axis 0 and assign it to the state_pool attribute - self.state_pool=np.expand_dims(s,axis=0) - # if a is an integer, execute the following statements - if type(a)==int: - # if the type of the first element of the param attribute of the nn attribute is not a list, execute the following statements - if type(self.nn.param[0])!=list: - # convert a to a numpy array with the dtype name of the first element of the param attribute of the nn attribute and assign it to a - a=np.array(a,self.nn.param[0].dtype.name) - # else, execute the following statements - else: - # convert a to a numpy array with the dtype name of the first element of the first element of the param attribute of the nn attribute and assign it to a - a=np.array(a,self.nn.param[0][0].dtype.name) - # expand the dimension of a along axis 0 and assign it to the action_pool attribute - self.action_pool=np.expand_dims(a,axis=0) - # else, execute the following statements - else: - # if the type of the first element of the param attribute of the nn attribute is not a list, execute the following statements - if type(self.nn.param[0])!=list: - # convert a to its dtype name with the first element of the param attribute of the nn attribute and assign it back to a - a=a.astype(self.nn.param[0].dtype.name) - # else, execute the following statements - else: - # convert a to its dtype name with the first element of the first element of the param attribute of the nn attribute and assign it back to a - a=a.astype(self.nn.param[0][0].dtype.name) - # assign a to the action_pool attribute - self.action_pool=a - # expand the dimension of next_s along axis 0 and assign it to the next_state_pool attribute - self.next_state_pool=np.expand_dims(next_s,axis=0) - # expand the dimension of r along axis 0 and assign it to the reward_pool attribute - self.reward_pool=np.expand_dims(r,axis=0) - # expand the dimension of done along axis 0 and assign it to the done_pool attribute - self.done_pool=np.expand_dims(done,axis=0) - # else, execute the following statements - else: - # append s to the state_pool attribute along axis 0 - self.state_pool=np.append(self.state_pool,np.expand_dims(s,axis=0),axis=0) - # if a is an integer, execute the following statements - if type(a)==int: - # if the type of the first element of the param attribute of the nn attribute is not a list, execute the following statements - if type(self.nn.param[0])!=list: - # convert a to a numpy array with the dtype name of the first element of the param attribute of the nn attribute and assign it to a - a=np.array(a,self.nn.param[0].dtype.name) - # else, execute the following statements - else: - # convert a to a numpy array with the dtype name of the first element of the first element of the param attribute of the nn attribute and assign it to a - a=np.array(a,self.nn.param[0][0].dtype.name) - # append a to the action_pool attribute along axis 0 - self.action_pool=np.append(self.action_pool,np.expand_dims(a,axis=0),axis=0) - # else, execute the following statements - else: - # if the type of the first element of the param attribute of the nn attribute is not a list, execute the following statements - if type(self.nn.param[0])!=list: - # convert a to its dtype name with the first element of the param attribute of the nn attribute and assign it back to a - a=a.astype(self.nn.param[0].dtype.name) - # else, execute the following statements - else: - # convert a to its dtype name with the first element of the first element of the param attribute of the nn attribute and assign it back to a - a=a.astype(self.nn.param[0][0].dtype.name) - # append a to the action_pool attribute along axis 0 - self.action_pool=np.append(self.action_pool,a,axis=0) - # append next_s to the next_state_pool attribute along axis 0 - self.next_state_pool=np.append(self.next_state_pool,np.expand_dims(next_s,axis=0),axis=0) - # append r to the reward_pool attribute along axis 0 - self.reward_pool=np.append(self.reward_pool,np.expand_dims(r,axis=0),axis=0) - # append done to the done_pool attribute along axis 0 - self.done_pool=np.append(self.done_pool,np.expand_dims(done,axis=0),axis=0) - # return nothing - return - - # define a method named choose_action - def choose_action(self,s): - # if hasattr(self.platform,'DType'): - if hasattr(self.platform,'DType'): - # expand dimension of s along axis 0 and assign it back to s - s=np.expand_dims(s,axis=0) - # assign output from actor method from nn with s as input to a variable and convert it to numpy array - a=(self.nn.actor.fp(s)+self.nn.noise()).numpy() - # else, execute following statements - else: - # expand dimension of s along axis 0 and assign it back to s - s=np.expand_dims(s,axis=0) - # convert s to tensor with float dtype and assign it to device attribute of nn and assign it back to s - s=self.platform.tensor(s,dtype=self.platform.float).to(self.nn.device) - # assign output from actor method from nn with s as input to a variable and convert it to numpy array and detach it from computation graph - a=(self.nn.actor(s)+self.nn.noise()).detach().numpy() - # if hasattr(self.nn,'discriminator'): - if hasattr(self.nn,'discriminator'): - # assign output from discriminator method from nn with s as input to reward variable and convert it to numpy array - reward=self.nn.discriminator(s).numpy() - # else, assign None to reward variable - else: - reward=None - # return a and reward - return a,reward - - # define a method named _train - def _train(self): - # if length of state_pool is less than batch, return numpy array of 0 - if len(self.state_pool)self.pool_size: - # assign a numpy array of 0 to the TD variable - TD=np.array(0) - # append TD to the TD attribute of the pr attribute of the nn attribute from index 2 onwards - self.nn.pr.TD=np.append(TD,self.nn.pr.TD[2:]) - # add r to the reward attribute and assign it back to the reward attribute - self.reward=r+self.reward - # assign the output of the _train method to the loss variable - loss=self._train() - # increment the sc attribute by 1 - self.sc+=1 - # if done is True, execute the following statements - if done: - # if the save_episode attribute is True, execute the following statements - if self.save_episode==True: - # append a list of s, a, next_s, r to the episode variable - episode=[s,a,next_s,r] - # append the reward attribute to the reward_list attribute - self.reward_list.append(self.reward) - # return loss, episode, done - return loss,episode,done - # else if save_episode is True, execute following statements - elif self.save_episode==True: - # append list of s, a, next_s, r to episode variable - episode=[s,a,next_s,r] - # assign next_s to s - s=next_s - # else, execute following statements - else: - # use while loop to iterate indefinitely - while True: - # call suspend_func method - self.suspend_func() - # assign output from epsilon_greedy_policy method with s as input to action_prob variable - action_prob=self.epsilon_greedy_policy(s) - # assign random choice from range of action_count with action_prob as probability to a variable - a=np.random.choice(range(self.action_count),p=action_prob) - # if nn has env attribute, execute following statements - if hasattr(self.nn,'env'): - # assign output from env method from nn with a as input to next_s, r, done variables - next_s,r,done=self.nn.env(a) - # else, execute following statements - else: - # assign output from step method from genv with a as input to next_s, r, done variables - next_s,r,done,_=self.genv.step(a) - # if hasattr(self.platform,'DType'): - if hasattr(self.platform,'DType'): - # if type of first element of param attribute of nn is not list, execute following statements - if type(self.nn.param[0])!=list: - # convert next_s to numpy array with dtype name of first element of param attribute of nn and assign it back to next_s - next_s=np.array(next_s,self.nn.param[0].dtype.name) - # convert r to numpy array with dtype name of first element of param attribute of nn and assign it back to r - r=np.array(r,self.nn.param[0].dtype.name) - # convert done to numpy array with dtype name of first element of param attribute of nn and assign it back to done - done=np.array(done,self.nn.param[0].dtype.name) - # else, execute following statements - else: - # convert next_s to numpy array with dtype name of first element of first element of param attribute of nn and assign it back to next_s - next_s=np.array(next_s,self.nn.param[0][0].dtype.name) - # convert r to numpy array with dtype name of first element of first element of param attribute of nn and assign it back to r - r=np.array(r,self.nn.param[0][0].dtype.name) - # convert done to numpy array with dtype name of first element of first element of param attribute of nn and assign it back to done - done=np.array(done,self.nn.param[0][0].dtype.name) - # if the nn attribute has a pool method, execute the following statements - if hasattr(self.nn,'pool'): - # call the pool method of the nn attribute with the state_pool, action_pool, next_state_pool, reward_pool, done_pool, s, a, next_s, r, done as inputs - self.nn.pool(self.state_pool,self.action_pool,self.next_state_pool,self.reward_pool,self.done_pool,[s,a,next_s,r,done]) - # else, execute the following statements - else: - # call the pool method with s, a, next_s, r, done as inputs - self.pool(s,a,next_s,r,done) - # if the nn attribute has a pr attribute, execute the following statements - if hasattr(self.nn,'pr'): - # append the initial_TD attribute of the nn attribute to the TD attribute of the pr attribute of the nn attribute - self.nn.pr.TD=np.append(self.nn.pr.TD,self.nn.initial_TD) - # if the length of the state_pool attribute is greater than the pool_size attribute, execute the following statements - if len(self.state_pool)>self.pool_size: - # assign a numpy array of 0 to the TD variable - TD=np.array(0) - # append TD to the TD attribute of the pr attribute of the nn attribute from index 2 onwards - self.nn.pr.TD=np.append(TD,self.nn.pr.TD[2:]) - # add r to the reward attribute and assign it back to the reward attribute - self.reward=r+self.reward - # assign the output of the _train method to the loss variable - loss=self._train() - # increment the sc attribute by 1 - self.sc+=1 - # if done is True or save_episode is True, execute the following statements - if done or self.save_episode==True: - # initialize episode variable as an empty list - episode=[] - # use a for loop to iterate over range from 0 to sc plus 1 - for i in range(0,self.sc+1): - # append a list of state_pool[i], action_pool[i], next_state_pool[i], reward_pool[i] to episode variable - episode.append([self.state_pool[i],self.action_pool[i],self.next_state_pool[i],self.reward_pool[i]]) - # if done is True, append 'done' string to episode variable - if done: - episode.append('done') - # return loss, episode, done - return loss,episode,done - # assign next_s to s - s=next_s - -# define a method named train -def train(self,episode_count,save=None,one=True,p=None,s=None): - # initialize the average reward variable as None - avg_reward=None - # if p is None, assign 9 to the p attribute - if p==None: - self.p=9 - # else, assign p minus 1 to the p attribute - else: - self.p=p-1 - # if s is None, assign 1 to the s attribute and None to the file_list attribute - if s==None: - self.s=1 - self.file_list=None - # else, assign s minus 1 to the s attribute and an empty list to the file_list attribute - else: - self.s=s-1 - self.file_list=[] - # if episode_count is not None, execute the following statements - if episode_count!=None: - # use a for loop to iterate from 0 to episode_count - for i in range(episode_count): - # record the current time as t1 - t1=time.time() - # call the train_ method and assign the outputs to loss, episode, and done variables - loss,episode,done=self.train_() - # if the trial_count attribute is not None, execute the following statements - if self.trial_count!=None: - # if the length of the reward_list attribute is greater than or equal to the trial_count attribute, execute the following statements - if len(self.reward_list)>=self.trial_count: - # calculate the mean of the last trial_count elements of the reward_list attribute and assign it to avg_reward - avg_reward=statistics.mean(self.reward_list[-self.trial_count:]) - # if the criterion attribute is not None and avg_reward is greater than or equal to the criterion attribute, execute the following statements - if self.criterion!=None and avg_reward>=self.criterion: - # record the current time as t2 - t2=time.time() - # add t2 minus t1 to the total_time attribute and assign it back to itself - self.total_time+=(t2-t1) - # assign total_time minus its integer part to time_ attribute - time_=self.total_time-int(self.total_time) - # if time_ is less than 0.5, assign the integer part of total_time to itself - if time_<0.5: - self.total_time=int(self.total_time) - # else, assign the integer part of total_time plus 1 to itself - else: - self.total_time=int(self.total_time)+1 - # print the total_episode attribute - print('episode:{0}'.format(self.total_episode)) - # print the loss with six decimal places - print('last loss:{0:.6f}'.format(loss)) - # print the avg_reward - print('average reward:{0}'.format(avg_reward)) - print() - # print the total_time with 's' as unit - print('time:{0}s'.format(self.total_time)) - # return nothing - return - # assign the loss to the loss attribute - self.loss=loss - # append the loss to the loss_list attribute - self.loss_list.append(loss) - # increment the total_episode attribute by 1 - self.total_episode+=1 - # if episode_count is not None and episode_count is divisible by 10, execute the following statements - if episode_count!=None and episode_count%10==0: - # assign episode_count divided by p plus 1 to p and convert it to an integer - p=int(episode_count/(self.p+1)) - # assign episode_count divided by s plus 1 to s and convert it to an integer - s=int(episode_count/(self.s+1)) - # else, execute the following statements - else: - # assign episode_count minus its remainder when divided by p to p and divide it by p and convert it to an integer - p=int((episode_count-episode_count%self.p)/self.p) - # assign episode_count minus its remainder when divided by s to s and divide it by s and convert it to an integer - s=int((episode_count-episode_count%s)/self.s) - # if p is 0, assign 1 to p - if p==0: - p=1 - # if s is 0, assign 1 to s - if s==0: - s=1 - # if i plus 1 is divisible by p, execute the following statements - if (i+1)%p==0: - # if the length of the state_pool attribute is greater than or equal to the batch attribute, execute the following statements - if len(self.state_pool)>=self.batch: - # print i plus 1 and loss with six decimal places - print('episode:{0} loss:{1:.6f}'.format(i+1,loss)) - # if avg_reward is not None, execute the following statements - if avg_reward!=None: - # print i plus 1 and avg_reward - print('episode:{0} average reward:{1}'.format(i+1,avg_reward)) - # else, execute the following statements - else: - # print i plus 1 and reward - print('episode:{0} reward:{1}'.format(i+1,self.reward)) - print() - # if save is not None and i plus 1 is divisible by s, execute the following statements - if save!=None and (i+1)%s==0: - # call the save method with total_episode and one as inputs - self.save(self.total_episode,one) - # if save_episode is True, execute the following statements - if self.save_episode==True: - # append episode to the episode_set attribute - self.episode_set.append(episode) - # if max_episode_count is not None and length of episode_set attribute is greater than or equal to max_episode_count attribute, execute the following statements - if self.max_episode_count!=None and len(self.episode_set)>=self.max_episode_count: - # assign False to save_episode attribute - self.save_episode=False - try: - try: - # call the assign_add method of the ec attribute of the nn attribute with 1 as input - self.nn.ec.assign_add(1) - except Exception: - # increment the ec attribute of the nn attribute by 1 - self.nn.ec+=1 - except Exception: - pass - t2=time.time() - self.time+=(t2-t1) - # else, execute the following statements - else: - # initialize i as 0 - i=0 - # use a while loop to iterate indefinitely - while True: - # record the current time as t1 - t1=time.time() - # call the train_ method and assign the outputs to loss, episode, and done variables - loss,episode,done=self.train_() - # if the trial_count attribute is not None, execute the following statements - if self.trial_count!=None: - # if the length of the reward_list attribute is equal to the trial_count attribute, execute the following statements - if len(self.reward_list)==self.trial_count: - # calculate the mean of the last trial_count elements of the reward_list attribute and assign it to avg_reward - avg_reward=statistics.mean(self.reward_list[-self.trial_count:]) - # if avg_reward is greater than or equal to the criterion attribute, execute the following statements - if avg_reward>=self.criterion: - # record the current time as t2 - t2=time.time() - # add t2 minus t1 to the total_time attribute and assign it back to itself - self.total_time+=(t2-t1) - # assign total_time minus its integer part to time_ attribute - time_=self.total_time-int(self.total_time) - # if time_ is less than 0.5, assign the integer part of total_time to itself - if time_<0.5: - self.total_time=int(self.total_time) - # else, assign the integer part of total_time plus 1 to itself - else: - self.total_time=int(self.total_time)+1 - # print the total_episode attribute - print('episode:{0}'.format(self.total_episode)) - # print the loss with six decimal places - print('last loss:{0:.6f}'.format(loss)) - # print the avg_reward - print('average reward:{0}'.format(avg_reward)) - print() - # print the total_time with 's' as unit - print('time:{0}s'.format(self.total_time)) - # return nothing - return - # assign the loss to the loss attribute - self.loss=loss - # append the loss to the loss_list attribute - self.loss_list.append(loss) - # increment i by 1 - i+=1 - # increment the total_episode attribute by 1 - self.total_episode+=1 - # if episode_count is not None and episode_count is not divisible by 10, execute the following statements - if episode_count!=None and episode_count%10!=0: - # assign episode_count minus its remainder when divided by p to p and divide it by p and convert it to an integer - p=int((episode_count-episode_count%self.p)/self.p) - # assign episode_count minus its remainder when divided by s to s and divide it by s and convert it to an integer - s=int((episode_count-episode_count%s)/self.s) - # else, execute the following statements - else: - # assign episode_count divided by p plus 1 to p and convert it to an integer - p=int(episode_count/(self.p+1)) - # assign episode_count divided by s plus 1 to s and convert it to an integer - s=int(episode_count/(self.s+1)) - # if p is 0, assign 1 to p - if p==0: - p=1 - # if s is 0, assign 1 to s - if s==0: - s=1 - # if i plus 1 is divisible by p, execute the following statements - if (i+1)%p==0: - # if the length of the state_pool attribute is greater than or equal to the batch attribute, execute the following statements - if len(self.state_pool)>=self.batch: - # print i plus 1 and loss with six decimal places - print('episode:{0} loss:{1:.6f}'.format(i+1,loss)) - # if avg_reward is not None, execute the following statements - if avg_reward!=None: - # print i plus 1 and avg_reward - print('episode:{0} average reward:{1}'.format(i+1,avg_reward)) - # else, execute the following statements - else: - # print i plus 1 and reward - print('episode:{0} reward:{1}'.format(i+1,self.reward)) - print() - # if save is not None and i plus 1 is divisible by s, execute the following statements - if save!=None and (i+1)%s==0: - # call the save method with total_episode and one as inputs - self.save(self.total_episode,one) - # if save_episode is True, execute the following statements - if self.save_episode==True: - # if done is True, append 'done' string to episode - if done: - episode.append('done') - # append episode to the episode_set attribute - self.episode_set.append(episode) - # if max_episode_count is not None and length of episode_set attribute is greater than or equal to max_episode_count attribute, execute the following statements - if self.max_episode_count!=None and len(self.episode_set)>=self.max_episode_count: - # assign False to save_episode attribute - self.save_episode=False - # if the nn attribute has an ec attribute, execute the following statements - if hasattr(self.nn,'ec'): - # try to execute the following statements - try: - # call the assign_add method of the ec attribute of the nn attribute with 1 as input - self.nn.ec.assign_add(1) - # if an exception occurs, execute the following statements - except Exception: - # increment the ec attribute of the nn attribute by 1 - self.nn.ec+=1 - # record the current time as t2 - t2=time.time() - # add t2 minus t1 to the time attribute and assign it back to itself - self.time+=(t2-t1) - # assign time minus its integer part to time_ attribute - time_=self.time-int(self.time) - # if time_ is less than 0.5, assign the integer part of time to itself - if time_<0.5: - self.total_time=int(self.time) - # else, assign the integer part of time plus 1 to itself - else: - self.total_time=int(self.time)+1 - # add time to the total_time attribute and assign it back to itself - self.total_time+=self.time - # print the loss with six decimal places - print('last loss:{0:.6f}'.format(loss)) - # print the reward - print('last reward:{0}'.format(self.reward)) - print() - # print the time with 's' as unit - print('time:{0}s'.format(self.time)) - # return nothing - return - - # define a method named train_online - def train_online(self): - # use while loop to iterate indefinitely - while True: - # if hasattr(nn,'save'): - if hasattr(self.nn,'save'): - # call save method of nn with save as input - self.nn.save(self.save) - # if hasattr(nn,'stop_flag'): - if hasattr(self.nn,'stop_flag'): - # if stop_flag attribute of nn is True, return nothing - if self.nn.stop_flag==True: - return - # if hasattr(nn,'stop_func'): - if hasattr(self.nn,'stop_func'): - # if output of stop_func method of nn is True, return nothing - if self.nn.stop_func(): - return - # if hasattr(nn,'suspend_func'): - if hasattr(self.nn,'suspend_func'): - # call suspend_func method of nn - self.nn.suspend_func() - # assign output from online method of nn to data variable - data=self.nn.online() - # if data is 'stop', return nothing - if data=='stop': - return - # elif data is 'suspend', execute following statements - elif data=='suspend': - # call suspend_func method of nn - self.nn.suspend_func() - # assign output from opt_ol method with first, second, third, fourth, fifth elements of data as inputs to loss variable - loss=self.opt_ol(data[0],data[1],data[2],data[3],data[4]) - # convert loss to numpy array and assign it back to loss - loss=loss.numpy() - # append loss to train_loss_list attribute of nn - self.nn.train_loss_list.append(loss) - # if length of train_acc_list attribute of nn equals max_length attribute of nn, delete first element of train_acc_list attribute of nn - if len(self.nn.train_acc_list)==self.nn.max_length: - del self.nn.train_acc_list[0] - # if hasattr(nn,'counter'): - if hasattr(self.nn,'counter'): - # increment counter attribute of nn by 1 - self.nn.counter+=1 - # return nothing - return From 454f1f2f0130a00fe62e36d2292802c47843239b Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Mon, 24 Jul 2023 12:55:00 +0800 Subject: [PATCH 079/337] Update nn.py --- Note 7.0 documentation/DL/neural network/pytorch/process/nn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Note 7.0 documentation/DL/neural network/pytorch/process/nn.py b/Note 7.0 documentation/DL/neural network/pytorch/process/nn.py index 7347d11d..56713cb0 100644 --- a/Note 7.0 documentation/DL/neural network/pytorch/process/nn.py +++ b/Note 7.0 documentation/DL/neural network/pytorch/process/nn.py @@ -28,7 +28,7 @@ def __init__(self): # define the constructor method self.device=torch.device('cpu') # otherwise set the device to cpu self.model=NeuralNetwork().to(self.device) # create an instance of the NeuralNetwork class and move it to the device self.loss_fn=nn.CrossEntropyLoss() # define the loss function as cross entropy loss - self.opt=[torch.optim.SGD(self.model.parameters(),lr=1e-3) for _ in range(3)] # define a list of three stochastic gradient descent optimizers with learning rate 1e-3 + self.opt=[torch.optim.SGD(self.model.parameters(),lr=1e-3) for _ in range(7)] # define a list of three stochastic gradient descent optimizers with learning rate 1e-3 def fp(self,x): # define a method for forward propagation From d17f1ee037d345ef23b15bcdb1afab520d5450a0 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Mon, 24 Jul 2023 12:55:17 +0800 Subject: [PATCH 080/337] Update nn_device.py --- .../DL/neural network/pytorch/process/nn_device.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Note 7.0 documentation/DL/neural network/pytorch/process/nn_device.py b/Note 7.0 documentation/DL/neural network/pytorch/process/nn_device.py index 4c890f52..ac58ebb6 100644 --- a/Note 7.0 documentation/DL/neural network/pytorch/process/nn_device.py +++ b/Note 7.0 documentation/DL/neural network/pytorch/process/nn_device.py @@ -30,7 +30,7 @@ def __init__(self): # define the constructor method self.device=torch.device('cpu') # set the device to cpu self.model=NeuralNetwork().to(self.device) # create an instance of the neural network model and move it to the device self.loss_fn=nn.CrossEntropyLoss() # define the loss function as cross entropy loss - self.opt=[torch.optim.SGD(self.model.parameters(),lr=1e-3) for _ in range(3)] # define a list of optimizers as stochastic gradient descent with learning rate 1e-3 + self.opt=[torch.optim.SGD(self.model.parameters(),lr=1e-3) for _ in range(7)] # define a list of optimizers as stochastic gradient descent with learning rate 1e-3 def fp(self,x,p): # define a method for forward propagation @@ -42,4 +42,4 @@ def fp(self,x,p): # define a method for forward propagation def loss(self,output,labels,p): # define a method for calculating loss loss=self.loss_fn(output,labels.to(assign_device_pytorch(p,'GPU'))) # assign the device according to the process index p # calculate the loss by passing the output and labels tensors to the device and then to the loss function - return loss # return the loss \ No newline at end of file + return loss # return the loss From a98914eb895abbba31b0d940d832880aef5435eb Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Mon, 24 Jul 2023 14:42:31 +0800 Subject: [PATCH 081/337] Update README.md --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index e51f4277..8844320f 100644 --- a/README.md +++ b/README.md @@ -353,7 +353,6 @@ kernel.set_up(epsilon=0.01,pool_size=10000,batch=64,update_step=10) #set up the kernel.PO=3 #use PO3 algorithm for parallel optimization pool_lock=[Lock(),Lock(),Lock(),Lock(),Lock()] #create a list of locks for each process's replay pool lock=[Lock(),Lock()] #create two locks for synchronization -g_lock=Lock() #create a global lock for gradient update for p in range(5): #loop over the processes Process(target=kernel.train,args=(p,100,lock,pool_lock)).start() #start each process with the train function and pass the process id, the number of episodes, the locks and the pool locks as arguments ``` @@ -370,7 +369,6 @@ kernel.set_up(epsilon=0.01,pool_size=10000,batch=64,update_step=10) #set up the kernel.PO=3 #use PO3 algorithm for parallel optimization pool_lock=[Lock(),Lock(),Lock(),Lock(),Lock()] #create a list of locks for each process's replay pool lock=[Lock(),Lock(),Lock()] #create a list of locks for synchronization -g_lock=Lock() #create a global lock for gradient update for p in range(5): #loop over the processes Process(target=kernel.train,args=(p,100,lock,pool_lock)).start() #start each process with the train function and pass the process id, the number of episodes, the locks and the pool locks as arguments ``` From 1f953f4ffec1ed76b4121edf68537c53e965001a Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Mon, 24 Jul 2023 15:17:45 +0800 Subject: [PATCH 082/337] Add files via upload --- .../RL/neural network/pytorch/process/DQN.py | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 Note 7.0 documentation/RL/neural network/pytorch/process/DQN.py diff --git a/Note 7.0 documentation/RL/neural network/pytorch/process/DQN.py b/Note 7.0 documentation/RL/neural network/pytorch/process/DQN.py new file mode 100644 index 00000000..d1f88712 --- /dev/null +++ b/Note 7.0 documentation/RL/neural network/pytorch/process/DQN.py @@ -0,0 +1,64 @@ +import torch +import gym +import torch.nn.functional as F + + +class Qnet(torch.nn.Module): + def __init__(self,state_dim,hidden_dim,action_dim): + super(Qnet,self).__init__() + self.fc1=torch.nn.Linear(state_dim,hidden_dim) + self.fc2=torch.nn.Linear(hidden_dim,action_dim) + + + def forward(self,x): + x=F.relu(self.fc1(x)) + return self.fc2(x) + + +class DQN: + def __init__(self,state_dim,hidden_dim,action_dim): + if torch.cuda.is_available(): + self.device=torch.device('cuda') + else: + self.device=torch.device('cpu') + self.nn=Qnet(state_dim,hidden_dim,action_dim).to(self.device) + self.target_q_net=Qnet(state_dim,hidden_dim,action_dim).to(self.device) + self.optimizer=[torch.optim.Adam(self.nn.parameters(),lr=2e-3) for _ in range(5)] #optimizer,kernel uses it to optimize. + self.genv=[gym.make('CartPole-v0') for _ in range(5)] #create environment + + + def env(self,a=None,p=None,initial=None): #environment function,kernel uses it to interact with the environment. + if initial==True: + state=self.genv[p].reset() + return state + else: + next_state,reward,done,_=self.genv[p].step(a) + return next_state,reward,done + + + def loss(self,s,a,next_s,r,d): #loss function,kernel uses it to calculate loss. + s=torch.tensor(s,dtype=torch.float).to(self.device) + a=torch.tensor(a,dtype=torch.int64).view(-1,1).to(self.device) + next_s=torch.tensor(next_s,dtype=torch.float).to(self.device) + r=torch.tensor(r,dtype=torch.float).view(-1,1).to(self.device) + d=torch.tensor(d,dtype=torch.float).view(-1,1).to(self.device) + q_value=self.nn(s).gather(1,a) + next_q_value=self.target_q_net(next_s).max(1)[0].view(-1,1) + target=r+0.98*next_q_value*(1-d) + return F.mse_loss(q_value,target) + + + def backward(self,loss,p): #backward function,kernel uses it for backpropagation. + self.optimizer[p].zero_grad(set_to_none=True) + loss.backward() + return + + + def opt(self,p): #opt function,kernel uses it to optimize. + self.optimizer[p].step() + return + + + def update_param(self): #update function,kernel uses it to update parameter. + self.target_q_net.load_state_dict(self.nn.state_dict()) + return \ No newline at end of file From e3db54f9c0bfea1c837ef228106633a825f80317 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Mon, 24 Jul 2023 19:02:47 +0800 Subject: [PATCH 083/337] Update nn_device.py --- .../DL/neural network/pytorch/process/nn_device.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Note 7.0 documentation/DL/neural network/pytorch/process/nn_device.py b/Note 7.0 documentation/DL/neural network/pytorch/process/nn_device.py index ac58ebb6..9c807390 100644 --- a/Note 7.0 documentation/DL/neural network/pytorch/process/nn_device.py +++ b/Note 7.0 documentation/DL/neural network/pytorch/process/nn_device.py @@ -1,6 +1,6 @@ import torch # import the PyTorch library from torch import nn # import the neural network module from PyTorch -from Note.nn.assign_device import assign_device_pytorch # import a custom function to assign device for PyTorch tensors +from Note.nn.process.assign_device import assign_device_pytorch # import a custom function to assign device for PyTorch tensors class NeuralNetwork(nn.Module): # define a class for the neural network model From 5a90e80ca399b35bd9b4f9c9a4712630efb02d92 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Mon, 24 Jul 2023 19:28:28 +0800 Subject: [PATCH 084/337] Update DQN.py --- .../RL/neural network/pytorch/process/DQN.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/Note 7.0 documentation/RL/neural network/pytorch/process/DQN.py b/Note 7.0 documentation/RL/neural network/pytorch/process/DQN.py index d1f88712..519cbef4 100644 --- a/Note 7.0 documentation/RL/neural network/pytorch/process/DQN.py +++ b/Note 7.0 documentation/RL/neural network/pytorch/process/DQN.py @@ -1,6 +1,7 @@ import torch import gym import torch.nn.functional as F +from Note.nn.process.assign_device_pytorch import assign_device class Qnet(torch.nn.Module): @@ -36,12 +37,12 @@ def env(self,a=None,p=None,initial=None): #environment function,kernel uses it t return next_state,reward,done - def loss(self,s,a,next_s,r,d): #loss function,kernel uses it to calculate loss. - s=torch.tensor(s,dtype=torch.float).to(self.device) - a=torch.tensor(a,dtype=torch.int64).view(-1,1).to(self.device) - next_s=torch.tensor(next_s,dtype=torch.float).to(self.device) - r=torch.tensor(r,dtype=torch.float).view(-1,1).to(self.device) - d=torch.tensor(d,dtype=torch.float).view(-1,1).to(self.device) + def loss(self,s,a,next_s,r,d,p): #loss function,kernel uses it to calculate loss. + s=torch.tensor(s,dtype=torch.float).to(assign_device(p,'GPU')) + a=torch.tensor(a,dtype=torch.int64).view(-1,1).to(assign_device(p,'GPU')) + next_s=torch.tensor(next_s,dtype=torch.float).to(assign_device(p,'GPU')) + r=torch.tensor(r,dtype=torch.float).view(-1,1).to(assign_device(p,'GPU')) + d=torch.tensor(d,dtype=torch.float).view(-1,1).to(assign_device(p,'GPU')) q_value=self.nn(s).gather(1,a) next_q_value=self.target_q_net(next_s).max(1)[0].view(-1,1) target=r+0.98*next_q_value*(1-d) @@ -61,4 +62,4 @@ def opt(self,p): #opt function,kernel uses it to optimize. def update_param(self): #update function,kernel uses it to update parameter. self.target_q_net.load_state_dict(self.nn.state_dict()) - return \ No newline at end of file + return From 3db8262bd8fde2ccbd954e00575de950873d7f3e Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Mon, 24 Jul 2023 19:28:54 +0800 Subject: [PATCH 085/337] Update and rename DQN.py to DQN.py --- .../RL/neural network/pytorch/{process => pool network}/DQN.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename Note 7.0 documentation/RL/neural network/pytorch/{process => pool network}/DQN.py (100%) diff --git a/Note 7.0 documentation/RL/neural network/pytorch/process/DQN.py b/Note 7.0 documentation/RL/neural network/pytorch/pool network/DQN.py similarity index 100% rename from Note 7.0 documentation/RL/neural network/pytorch/process/DQN.py rename to Note 7.0 documentation/RL/neural network/pytorch/pool network/DQN.py From 130896d3ff8dcbbfff76a42d7ae62491979b895a Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Mon, 24 Jul 2023 19:29:11 +0800 Subject: [PATCH 086/337] Update and rename DQN.py to DQN.py --- .../neural network/tensorflow/{pool net => pool network}/DQN.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename Note 7.0 documentation/RL/neural network/tensorflow/{pool net => pool network}/DQN.py (100%) diff --git a/Note 7.0 documentation/RL/neural network/tensorflow/pool net/DQN.py b/Note 7.0 documentation/RL/neural network/tensorflow/pool network/DQN.py similarity index 100% rename from Note 7.0 documentation/RL/neural network/tensorflow/pool net/DQN.py rename to Note 7.0 documentation/RL/neural network/tensorflow/pool network/DQN.py From 6afdd7931dca4b223fee23dfe1e3f5378585c4f3 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Mon, 24 Jul 2023 19:33:03 +0800 Subject: [PATCH 087/337] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8844320f..6dfb015b 100644 --- a/README.md +++ b/README.md @@ -318,7 +318,7 @@ for p in range(7): #loop over the processes **You can download neural network example in this link,and then you can import neural network and train with kernel,link and example code are below.** -https://github.com/NoteDancing/Note-documentation/blob/Note-7.0/Note%207.0%20documentation/RL/neural%20network/tensorflow/pool%20net/DQN.py +https://github.com/NoteDancing/Note-documentation/blob/Note-7.0/Note%207.0%20documentation/RL/neural%20network/tensorflow/pool%20network/DQN.py ### PO2: ```python From fffe1f9c30502e893fe3837ff5d99e21af68d559 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Tue, 25 Jul 2023 18:11:51 +0800 Subject: [PATCH 088/337] Add files via upload --- .../RL/kernel/kernel_pytorch.py | 463 ++++++++++++++++++ 1 file changed, 463 insertions(+) create mode 100644 Note 7.0 documentation/RL/kernel/kernel_pytorch.py diff --git a/Note 7.0 documentation/RL/kernel/kernel_pytorch.py b/Note 7.0 documentation/RL/kernel/kernel_pytorch.py new file mode 100644 index 00000000..89248627 --- /dev/null +++ b/Note 7.0 documentation/RL/kernel/kernel_pytorch.py @@ -0,0 +1,463 @@ +import torch +from multiprocessing import Value,Array +from Note.nn.process.assign_device_pytorch import assign_device +import numpy as np +import statistics + + +class kernel: + def __init__(self,nn=None,process=None,device='GPU'): + self.nn=nn # the neural network model for the agent + if process!=None: + self.reward=np.zeros(process,dtype=np.float32) # the reward array for each process + self.sc=np.zeros(process,dtype=np.int32) # the step counter array for each process + self.device=device # the device to run the model on, either GPU or CPU + self.state_pool={} # the dictionary to store the state transitions for each process + self.action_pool={} # the dictionary to store the actions for each process + self.next_state_pool={} # the dictionary to store the next states for each process + self.reward_pool={} # the dictionary to store the rewards for each process + self.done_pool={} # the dictionary to store the done flags for each process + self.epsilon=None # the epsilon value for the epsilon-greedy policy + self.episode_step=None # the maximum number of steps per episode + self.pool_size=None # the maximum size of the state pool for each process + self.batch=None # the batch size for training + self.episode_=0 # the episode counter + self.update_step=None # the frequency of updating the target network parameters + self.trial_count=None # the number of trials to calculate the average reward + self.process=process # the number of processes to run in parallel + self.process_counter=0 # the counter of running processes + self.probability_list=[] # the list of probabilities to select a process + self.running_flag_list=[] # the list of flags to indicate whether a process is running or not + self.finish_list=[] # the list of indices of finished processes + self.running_flag=[] # the list of flags to indicate whether a process is running or not (shared) + self.priority_flag=False # the flag to indicate whether to use priority-based optimization or not + self.priority_p=0 # the index of the priority process to optimize first + self.max_opt=None # the maximum number of optimization steps for a priority process + self.stop=False # the flag to indicate whether to stop training or not + self.save_flag=False # the flag to indicate whether to save the model parameters or not + self.stop_flag=False # the flag to indicate whether to stop training or not (shared) + self.opt_counter=None # the array to store the optimization counter for each process + self.s=None # a temporary variable to store a state tensor + self.filename='save.dat' # the file name to save and load the model parameters and states + self.reward_list=[] # the list of rewards for each episode (shared) + self.loss_list=[] # the list of losses for each episode (shared) + self.total_episode=0 # the total number of episodes (shared) + + + def init(self,manager): + """This method is used to initialize some shared variables using a manager object""" + + self.state_pool=manager.dict(self.state_pool) + self.action_pool=manager.dict(self.action_pool) + self.next_state_pool=manager.dict(self.next_state_pool) + self.reward_pool=manager.dict(self.reward_pool) + self.done_pool=manager.dict(self.done_pool) + self.reward=Array('f',self.reward) + self.loss=np.zeros(self.process) + self.loss=Array('f',self.loss) + self.sc=Array('i',self.sc) + self.process_counter=Value('i',self.process_counter) + self.probability_list=manager.list(self.probability_list) + self.running_flag_list=manager.list(self.running_flag_list) + self.finish_list=manager.list(self.finish_list) + self.running_flag=manager.list([0]) + self.reward_list=manager.list(self.reward_list) + self.loss_list=manager.list(self.loss_list) + self.total_episode=Value('i',self.total_episode) + self.priority_p=Value('i',self.priority_p) + if self.priority_flag==True: + self.opt_counter=Array('i',np.zeros(self.process,dtype=np.int32)) + try: + self.nn.opt_counter=manager.list([self.nn.opt_counter]) + except Exception: + self.opt_counter_=manager.list() + try: + self.nn.ec=manager.list([self.nn.ec]) + except Exception: + self.ec_=manager.list() + try: + self.nn.bc=manager.list([self.nn.bc]) + except Exception: + self.bc_=manager.list() + self.episode_=Value('i',self.total_episode.value) + self.stop_flag=Value('b',self.stop_flag) + self.save_flag=Value('b',self.save_flag) + self.file_list=manager.list([]) + return + + + def init_online(self,manager): + """This method is used to initialize some shared variables for online learning using a manager object""" + + self.nn.train_loss_list=manager.list([]) + self.nn.counter=manager.list([]) + self.nn.exception_list=manager.list([]) + return + + + def action_vec(self): + """This method is used to create a vector of ones with the same length as the action space""" + + self.action_one=np.ones(self.action_count,dtype=np.int8) + return + + + def set_up(self,epsilon=None,episode_step=None,pool_size=None,batch=None,update_step=None,trial_count=None,criterion=None): + """This method is used to set up some hyperparameters for the agent""" + + if epsilon!=None: + self.epsilon=np.ones(self.process)*epsilon # set the epsilon value for each process + if episode_step!=None: + self.episode_step=episode_step # set the maximum number of steps per episode + if pool_size!=None: + self.pool_size=pool_size # set the maximum size of the state pool for each process + if batch!=None: + self.batch=batch # set the batch size for training + if update_step!=None: + self.update_step=update_step # set the frequency of updating the target network parameters + if trial_count!=None: + self.trial_count=trial_count # set the number of trials to calculate the average reward + if criterion!=None: + self.criterion=criterion # set the criterion to stop training when the average reward reaches it + if epsilon!=None: + self.action_vec() # create a vector of ones with the same length as the action space + return + + + def epsilon_greedy_policy(self,s,epsilon,p): + """This method is used to implement an epsilon-greedy policy for action selection""" + + action_prob=self.action_one*epsilon/len(self.action_one) # initialize a uniform distribution over actions + s=torch.tensor(s,dtype=torch.float).to(assign_device(p,self.device)) # convert the state to a tensor and assign it to a device + best_a=self.nn.nn(s).argmax() # get the best action according to the network output + action_prob[best_a.numpy()]+=1-epsilon # increase the probability of the best action by 1-epsilon + return action_prob # return the action probability vector + + + def pool(self,s,a,next_s,r,done,pool_lock,index): + """This method is used to store the state transitions in the state pool for a given process index""" + + pool_lock[index].acquire() # acquire a lock for the process index + try: + if type(self.state_pool[index])!=np.ndarray and self.state_pool[index]==None: # if the state pool is empty for this index + self.state_pool[index]=s # store the state as an array + if type(a)==int: # if the action is an integer + a=np.array(a) # convert it to an array + self.action_pool[index]=np.expand_dims(a,axis=0) # store the action as an array with an extra dimension + else: # if the action is already an array + self.action_pool[index]=a # store the action as it is + self.next_state_pool[index]=np.expand_dims(next_s,axis=0) # store the next state as an array with an extra dimension + self.reward_pool[index]=np.expand_dims(r,axis=0) # store the reward as an array with an extra dimension + self.done_pool[index]=np.expand_dims(done,axis=0) # store the done flag as an array with an extra dimension + else: # if the state pool is not empty for this index + try: + self.state_pool[index]=np.concatenate((self.state_pool[index],s),0) # append the state to the existing state pool + if type(a)==int: # if the action is an integer + a=np.array(a) # convert it to an array + self.action_pool[index]=np.concatenate((self.action_pool[index],np.expand_dims(a,axis=0)),0) # append the action to the existing action pool with an extra dimension + else: # if the action is already an array + self.action_pool[index]=np.concatenate((self.action_pool[index],a),0) # append the action to the existing action pool as it is + self.next_state_pool[index]=np.concatenate((self.next_state_pool[index],np.expand_dims(next_s,axis=0)),0) # append the next state to the existing next state pool with an extra dimension + self.reward_pool[index]=np.concatenate((self.reward_pool[index],np.expand_dims(r,axis=0)),0) # append the reward to the existing reward pool with an extra dimension + self.done_pool[index]=np.concatenate((self.done_pool[index],np.expand_dims(done,axis=0)),0) # append the done flag to the existing done pool with an extra dimension + except Exception: + pass + if type(self.state_pool[index])==np.ndarray and len(self.state_pool[index])>self.pool_size: # if the state pool exceeds the maximum size for this index + self.state_pool[index]=self.state_pool[index][1:] # remove the oldest state from the state pool + self.action_pool[index]=self.action_pool[index][1:] # remove the oldest action from the action pool + self.next_state_pool[index]=self.next_state_pool[index][1:] # remove the oldest next state from the next state pool + self.reward_pool[index]=self.reward_pool[index][1:] # remove the oldest reward from the reward pool + self.done_pool[index]=self.done_pool[index][1:] # remove the oldest done flag from the done pool + except Exception: + pool_lock[index].release() # release the lock for the process index + return + pool_lock[index].release() # release the lock for the process index + return + + + def get_index(self,p,lock): + """This method is used to get a random index of a running process according to their probabilities""" + + while len(self.running_flag_list)self.process_counter.value: # if there are not enough or too many running flags for this process index + self.running_flag_list[p]=self.running_flag[1:].copy() # update the running flag list for this process index with a copy of the running flag list (without the first element) + while len(self.probability_list)=self.trial_count: # if there are enough rewards in the list + avg_reward=statistics.mean(self.reward_list[-self.trial_count:]) # calculate the average reward of the last trial count episodes + if self.criterion!=None and avg_reward>=self.criterion: # if there is a criterion for stopping and the average reward reaches it + return True # return True to indicate that training should be stopped + return False # return False to indicate that training should not be stopped + + + def opt(self,state_batch,action_batch,next_state_batch,reward_batch,done_batch,p): + """This method is used to optimize the network parameters using a batch of data""" + + loss=self.nn.loss(state_batch,action_batch,next_state_batch,reward_batch,done_batch,p) # calculate the loss according to the network model and the data batch + if self.priority_flag==True and self.priority_p.value!=-1: # if priority-based optimization is enabled and there is a priority process index + while True: + if self.stop_flag.value==True: # if training should be stopped (shared) + return None # return None to indicate that optimization is aborted + if p==self.priority_p.value: # if this process index is equal to the priority process index + break # break out of the loop + else: # if this process index is not equal to the priority process index + continue # try again + if self.stop_func_(): # if training should be stopped (local) + return None # return None to indicate that optimization is aborted + loss=loss.clone() # clone the loss tensor to avoid modifying it in place + self.nn.backward(loss,p) # perform backpropagation on the loss tensor according to the network model and this process index + self.nn.opt(p) # perform optimization on the network parameters according to this process index + return loss # return the loss tensor + + + def _train(self,p,j,batches,length): + """This method is used to train on a batch of data from the state pool for a given process index""" + + if j==batches-1: # if this is the last batch of data + index1=batches*self.batch # get the starting index of this batch from the end of the state pool + index2=self.batch-(length-batches*self.batch) # get the ending index of this batch from the beginning of the state pool + state_batch=np.concatenate((self.state_pool[p][index1:length],self.state_pool[p][:index2]),0) # concatenate two slices of states from both ends of the state pool as a batch + action_batch=np.concatenate((self.action_pool[p][index1:length],self.action_pool[p][:index2]),0) # concatenate two slices of actions from both ends of the action pool as a batch + next_state_batch=np.concatenate((self.next_state_pool[p][index1:length],self.next_state_pool[p][:index2]),0) # concatenate two slices of next states from both ends of the next state pool as a batch + reward_batch=np.concatenate((self.reward_pool[p][index1:length],self.reward_pool[p][:index2]),0) # concatenate two slices of rewards from both ends of the reward pool as a batch + done_batch=np.concatenate((self.done_pool[p][index1:length],self.done_pool[p][:index2]),0) # concatenate two slices of done flags from both ends of the done pool as a batch + loss=self.opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch,p) # optimize the network parameters using this batch of data + self.loss[p]+=loss # accumulate the loss for this process index + if hasattr(self.nn,'bc'): # if there is a batch counter for the network model + bc=self.nn.bc[0] # get the current value of the batch counter + bc.assign_add(1) # increment the batch counter by 1 + self.nn.bc[0]=bc # update the batch counter for the network model + else: # if this is not the last batch of data + index1=j*self.batch # get the starting index of this batch from the state pool + index2=(j+1)*self.batch # get the ending index of this batch from the state pool + state_batch=self.state_pool[p][index1:index2] # get a slice of states from the state pool as a batch + action_batch=self.action_pool[p][index1:index2] # get a slice of actions from the action pool as a batch + next_state_batch=self.next_state_pool[p][index1:index2] # get a slice of next states from the next state pool as a batch + reward_batch=self.reward_pool[p][index1:index2] # get a slice of rewards from the reward pool as a batch + done_batch=self.done_pool[p][index1:index2] # get a slice of done flags from the done pool as a batch + loss=self.opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch,p) # optimize the network parameters using this batch of data + self.loss[p]+=loss # accumulate the loss for this process index + if hasattr(self.nn,'bc'): # if there is a batch counter for the network model + bc=self.nn.bc[0] # get the current value of the batch counter + bc.assign_add(1) # increment the batch counter by 1 + self.nn.bc[0]=bc # update the batch counter for the network model + return + + + def train_(self,p,lock): + """This method is used to train on all batches of data from the state pool for a given process index""" + + if len(self.done_pool[p])=self.max_opt: # if there is a maximum number of optimization steps for a priority process and it is reached by the priority process index + self.priority_p.value=int(self.priority_p.value) # convert the priority process index to an integer type + elif self.max_opt==None: # if there is no maximum number of optimization steps for a priority process + self.priority_p.value=int(self.priority_p.value) # convert the priority process index to an integer type + else: # if there is a maximum number of optimization steps for a priority process and it is not reached by any process index + self.priority_p.value=-1 # set the priority process index to -1 to indicate no priority + if self.priority_flag==True: # if priority-based optimization is enabled + self.opt_counter[p]=0 # reset the optimization counter for this process index + if hasattr(self.nn,'attenuate'): # if there is an attenuate method for the network model + self.nn.attenuate(p) # attenuate some parameters according to the network model and this process index + self._train(p,j,batches,length) # train on a batch of data from the state pool for this process index + if self.priority_flag==True: # if priority-based optimization is enabled + opt_counter=np.frombuffer(self.opt_counter.get_obj(),dtype='i') # get the optimization counter array as a numpy array + opt_counter+=1 # increment the optimization counter array by 1 + if hasattr(self.nn,'attenuate'): # if there is an attenuate method for the network model + opt_counter=self.nn.opt_counter[0] # get the current value of the optimization counter for the network model + opt_counter+=1 # increment the optimization counter for the network model by 1 + self.nn.opt_counter[0]=opt_counter # update the optimization counter for the network model + if self.update_step!=None: # if there is a frequency of updating the target network parameters + if self.sc[p]%self.update_step==0: # if this process index reaches the update frequency + self.nn.update_param() # update the target network parameters according to the network model + else: # if there is no frequency of updating the target network parameters + self.nn.update_param() # update the target network parameters according to the network model + self.loss[p]=self.loss[p]/batches # calculate the average loss for this process index + self.sc[p]+=1 # increment the step counter for this process index + if hasattr(self.nn,'ec'): # if there is an episode counter for the network model + ec=self.nn.ec[0] # get the current value of the episode counter + ec.assign_add(1) # increment the episode counter by 1 + self.nn.ec[0]=ec # update the episode counter for the network model + return + + + def train(self,p,episode_count,lock,pool_lock): + """This method is used to start a process to train on multiple episodes""" + + lock[1].acquire() # acquire a lock for the shared variables + self.state_pool[p]=None # initialize the state pool for this process index as None + self.action_pool[p]=None # initialize the action pool for this process index as None + self.next_state_pool[p]=None # initialize the next state pool for this process index as None + self.reward_pool[p]=None # initialize the reward pool for this process index as None + self.done_pool[p]=None # initialize the done pool for this process index as None + self.running_flag.append(1) # append a 1 to the running flag list to indicate that this process is running + self.process_counter.value+=1 # increment the counter of running processes by 1 + self.finish_list.append(None) # append a None to the finish list to indicate that this process is not finished yet + try: + epsilon=self.epsilon[p] # get the epsilon value for this process index + except Exception: + epsilon=None # set the epsilon value to None if there is no epsilon value + lock[1].release() # release the lock for the shared variables + for k in range(episode_count): # for each episode + s=self.nn.env(p=p,initial=True) # get an initial state from the environment according to this process index + s=np.array(s) # convert the state to an array + if self.episode_step==None: # if there is no maximum number of steps per episode + while True: # loop until done + next_s,r,done,index=self.env(s,epsilon,p,lock,pool_lock) # interact with the environment and store the state transitions in the state pool + self.reward[p]+=r # accumulate the reward for this process index + s=next_s # update the state with next state + if type(self.done_pool[p])==np.ndarray: # if there are enough data in the state pool for this process index + self.train_(p,lock) # train on all batches of data from the state pool for this process index + if done: # if done flag is True + if len(lock)==4: # if there is a fourth lock for saving and loading + lock[3].acquire() # acquire a lock for saving and loading + self.total_episode.value+=1 # increment the total number of episodes by 1 (shared) + self.loss_list.append(self.loss[p]) # append the loss for this process index to the loss list (shared) + if len(lock)==4: # if there is a fourth lock for saving and loading + lock[3].acquire() # acquire a lock for saving and loading + self.total_episode.value+=1 # increment the total number of episodes by 1 (shared) + self.loss_list.append(self.loss[p]) # append the loss for this process index to the loss list (shared) + if len(lock)==4: # if there is a fourth lock for saving and loading + lock[3].release() # release the lock for saving and loading + break # break out of the loop + else: # if there is a maximum number of steps per episode + for l in range(self.episode_step): # for each step + next_s,r,done,index=self.env(s,epsilon,p,lock,pool_lock) # interact with the environment and store the state transitions in the state pool + self.reward[p]+=r # accumulate the reward for this process index + s=next_s # update the state with next state + if type(self.done_pool[p])==np.ndarray: # if there are enough data in the state pool for this process index + self.train_(p,lock) # train on all batches of data from the state pool for this process index + if done: # if done flag is True + if len(lock)==4: # if there is a fourth lock for saving and loading + lock[3].acquire() # acquire a lock for saving and loading + self.total_episode.value+=1 # increment the total number of episodes by 1 (shared) + self.loss_list.append(self.loss[p]) # append the loss for this process index to the loss list (shared) + if len(lock)==4: # if there is a fourth lock for saving and loading + lock[3].release() # release the lock for saving and loading + break # break out of the loop + if l==self.episode_step-1: # if this is the last step of the episode + if len(lock)==4: # if there is a fourth lock for saving and loading + lock[3].acquire() # acquire a lock for saving and loading + self.total_episode.value+=1 # increment the total number of episodes by 1 (shared) + self.loss_list.append(self.loss[p]) # append the loss for this process index to the loss list (shared) + if len(lock)==4: # if there is a fourth lock for saving and loading + lock[3].release() # release the lock for saving and loading + if len(lock)==3 or len(lock)==4: # if there is a third or fourth lock for saving and loading + lock[2].acquire() # acquire a lock for saving and loading + self.save_() # save the model parameters and states according to the network model + self.reward_list.append(self.reward[p]) # append the reward for this process index to the reward list (shared) + self.reward[p]=0 # reset the reward for this process index + if len(lock)==3 or len(lock)==4: # if there is a third or fourth lock for saving and loading + lock[2].release() # release the lock for saving and loading + self.running_flag[p+1]=0 # set the running flag to 0 to indicate that this process is not running anymore + if p not in self.finish_list: # if this process index is not in the finish list yet + self.finish_list[p]=p # add this process index to the finish list + lock[1].acquire() # acquire a lock for the shared variables + self.process_counter.value-=1 # decrement the counter of running processes by 1 + lock[1].release() # release the lock for the shared variables + del self.state_pool[p] # delete the state pool for this process index + del self.action_pool[p] # delete the action pool for this process index + del self.next_state_pool[p] # delete the next state pool for this process index + del self.reward_pool[p] # delete the reward pool for this process index + del self.done_pool[p] # delete the done pool for this process index + return + + + def train_online(self,p,lock=None,g_lock=None): + """This method is used to implement online learning using a single process""" + + if hasattr(self.nn,'counter'): # if there is a counter variable for the network model + self.nn.counter.append(0) # append a 0 to the counter list + while True: # loop until stopped + if hasattr(self.nn,'save'): # if there is a save method for the network model + self.nn.save(self.save,p) # save the model parameters and states according to the network model and this process index + if hasattr(self.nn,'stop_flag'): # if there is a stop flag for the network model + if self.nn.stop_flag==True: # if the stop flag is True + return # return to indicate that online learning is stopped + if hasattr(self.nn,'stop_func'): # if there is a stop function for the network model + if self.nn.stop_func(p): # if the stop function returns True according to this process index + return # return to indicate that online learning is stopped + if hasattr(self.nn,'suspend_func'): # if there is a suspend function for the network model + self.nn.suspend_func(p) # suspend some operations according to the network model and this process index + try: + data=self.nn.online(p) # get some data from the online source according to the network model and this process index + except Exception as e: # if there is an exception raised + self.nn.exception_list[p]=e # append the exception to the exception list for this process index + if data=='stop': # if the data indicates to stop online learning + return # return to indicate that online learning is stopped + elif data=='suspend': # if the data indicates to suspend some operations + self.nn.suspend_func(p) # suspend some operations according to the network model and this process index + try: + loss,param=self.opt(data[0],data[1],data[2],data[3],data[4],p,lock,g_lock) # optimize the network parameters using the data and some locks + except Exception as e: # if there is an exception raised + self.nn.exception_list[p]=e # append the exception to the exception list for this process index + loss=loss.numpy() # convert the loss tensor to a numpy array + if len(self.nn.train_loss_list)==self.nn.max_length: # if the train loss list reaches its maximum length + del self.nn.train_loss_list[0] # delete the oldest element from the train loss list + self.nn.train_loss_list.append(loss) # append the loss to the train loss list + try: + if hasattr(self.nn,'counter'): # if there is a counter variable for the network model + count=self.nn.counter[p] # get the current value of the counter for this process index + count+=1 # increment the counter by 1 + self.nn.counter[p]=count # update the counter for this process index + except IndexError: # if there is no counter for this process index yet + self.nn.counter.append(0) # append a 0 to the counter list + count=self.nn.counter[p] # get the current value of the counter for this process index + count+=1 # increment the counter by 1 + self.nn.counter[p]=count # update the counter for this process index + return \ No newline at end of file From e833586d80c4f90b297db9abd2e79c671bbcce41 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Tue, 25 Jul 2023 19:55:44 +0800 Subject: [PATCH 089/337] Add files via upload --- .../RL/kernel/nspn/kernel.py | 586 ++++++++++++++++++ 1 file changed, 586 insertions(+) create mode 100644 Note 7.0 documentation/RL/kernel/nspn/kernel.py diff --git a/Note 7.0 documentation/RL/kernel/nspn/kernel.py b/Note 7.0 documentation/RL/kernel/nspn/kernel.py new file mode 100644 index 00000000..e36f68e2 --- /dev/null +++ b/Note 7.0 documentation/RL/kernel/nspn/kernel.py @@ -0,0 +1,586 @@ +import tensorflow as tf +import numpy as np +import statistics +import time + + +class kernel: + # This is the constructor method for initializing the class attributes + def __init__(self,nn=None,save_episode=False): + self.nn=nn # This is the neural network object + if hasattr(self.nn,'km'): # This is a condition to check if the model has a km attribute + self.nn.km=1 # This is to assign the value 1 to the km attribute + self.platform=None # This is the platform for running the algorithm, such as TensorFlow or PyTorch + self.state_pool=None # This is a pool for storing the states from the environment + self.action_pool=None # This is a pool for storing the actions taken by the agent + self.next_state_pool=None # This is a pool for storing the next states from the environment + self.reward_pool=None # This is a pool for storing the rewards received by the agent + self.done_pool=None # This is a pool for storing the done flags indicating the end of an episode + self.episode_set=[] # This is a list for storing the episodes generated by the agent + self.epsilon=None # This is a parameter for controlling the exploration-exploitation trade-off + self.episode_step=None # This is a parameter for limiting the number of steps per episode + self.pool_size=None # This is a parameter for limiting the size of the pools + self.batch=None # This is a parameter for determining the batch size for training + self.update_step=None # This is a parameter for determining the frequency of updating the model parameters + self.trial_count=None # This is a parameter for determining the number of trials for evaluating the performance + self.criterion=None # This is a parameter for determining the criterion for stopping the training + self.reward_list=[] # This is a list for storing the rewards per episode + self.suspend=False # This is a flag for indicating whether to suspend the training or not + self.save_epi=None # This is a parameter for determining whether to save the episodes or not + self.max_episode_count=None # This is a parameter for limiting the number of episodes to save + self.save_episode=save_episode # This is an argument for setting the save_epi parameter + self.filename='save.dat' # This is a filename for saving or loading the data + self.loss=None # This is a variable for storing the loss value per batch + self.loss_list=[] # This is a list for storing the loss values per episode + self.sc=0 # This is a counter for counting the number of steps taken by the agent + self.total_episode=0 # This is a counter for counting the number of episodes completed by the agent + self.time=0 # This is a variable for measuring the time elapsed per episode + self.total_time=0 # This is a variable for measuring the total time elapsed + + # This is a method for creating an action vector based on epsilon-greedy policy + def action_vec(self): + if self.epsilon!=None: # This is a condition to check if epsilon is defined or not + self.action_one=np.ones(self.action_count,dtype=np.int8) # This is to create an array of ones with length equal to action_count, which is the number of possible actions in the environment + return + + + # This is a method for initializing or resetting some of the class attributes + def init(self): + try: + if hasattr(self.nn,'pr'): # This is a condition to check if the model has a pr attribute, which stands for prioritized replay buffer + self.nn.pr.TD=np.array(0) # This is to assign an array of zero to the TD attribute, which stands for temporal difference error + except Exception as e: # This is to catch any exception that may occur during this process + raise e # This is to raise or re-throw the exception + self.suspend=False # This is to reset the suspend flag to False + self.save_epi=None # This is to reset the save_epi parameter to None + self.episode_set=[] # This is to reset the episode_set list to an empty list + self.state_pool=None # This is to reset the state_pool to None + self.action_pool=None # This is to reset the action_pool to None + self.next_state_pool=None # This is to reset the next_state_pool to None + self.reward_pool=None # This is to reset the reward_pool to None + self.done_pool=None # This is to reset the done_pool to None + self.reward_list=[] # This is to reset the reward_list to an empty list + self.loss=0 # This is to reset the loss value to zero + self.loss_list=[] # This is to reset the loss_list to an empty list + self.sc=0 # This is to reset the step counter to zero + self.total_episode=0 # This is to reset the episode counter to zero + self.time=0 # This is to reset the time variable to zero + self.total_time=0 # This is to reset the total_time variable to zero + return + + + # This is a method for setting up some of the class attributes based on the arguments + def set_up(self,epsilon=None,episode_step=None,pool_size=None,batch=None,update_step=None,trial_count=None,criterion=None): + if epsilon!=None: # This is a condition to check if epsilon is given or not + self.epsilon=epsilon # This is to assign the epsilon value to the epsilon attribute + if episode_step!=None: # This is a condition to check if episode_step is given or not + self.episode_step=episode_step # This is to assign the episode_step value to the episode_step attribute + if pool_size!=None: # This is a condition to check if pool_size is given or not + self.pool_size=pool_size # This is to assign the pool_size value to the pool_size attribute + if batch!=None: # This is a condition to check if batch is given or not + self.batch=batch # This is to assign the batch value to the batch attribute + if update_step!=None: # This is a condition to check if update_step is given or not + self.update_step=update_step # This is to assign the update_step value to the update_step attribute + if trial_count!=None: # This is a condition to check if trial_count is given or not + self.trial_count=trial_count # This is to assign the trial_count value to the trial_count attribute + if criterion!=None: # This is a condition to check if criterion is given or not + self.criterion=criterion # This is to assign the criterion value to the criterion attribute + self.action_vec() # This is to call the action_vec method for creating an action vector based on epsilon-greedy policy + return + + + # This is a method for implementing an epsilon-greedy policy for choosing an action based on a state + def epsilon_greedy_policy(self,s): + action_prob=self.action_one*self.epsilon/len(self.action_one) # This is to create an array of probabilities for each action based on epsilon and action_one, which are class attributes + try: + if hasattr(self.platform,'DType'): # This is a condition to check if the platform has a DType attribute, which indicates that it is TensorFlow + best_a=np.argmax(self.nn.nn.fp(s)) # This is to find the best action based on the neural network model's output for the state s using fp method, which stands for forward propagation + action_prob[best_a]+=1-self.epsilon # This is to increase the probability of choosing the best action by 1-epsilon + else: # This means that the platform does not have a DType attribute, which indicates that it is PyTorch + s=self.platform.tensor(s,dtype=self.platform.float).to(self.nn.device) # This is to convert the state s into a PyTorch tensor with float data type and send it to the device where the model runs, such as CPU or GPU + best_a=self.nn.nn(s).argmax() # This is to find the best action based on the neural network model's output for the state s using nn method, which stands for neural network + action_prob[best_a.numpy()]+=1-self.epsilon # This is to increase the probability of choosing the best action by 1-epsilon and convert it into a numpy array + except Exception as e: # This is to catch any exception that may occur during this process + raise e # This is to raise or re-throw the exception + return action_prob + + + # This is a method for computing the loss and updating the model parameters using TensorFlow + @tf.function(jit_compile=True) + def tf_opt(self,state_batch,action_batch,next_state_batch,reward_batch,done_batch): + with self.platform.GradientTape(persistent=True) as tape: # This is to create a gradient tape object for recording and computing gradients in TensorFlow + loss=self.nn.loss(state_batch,action_batch,next_state_batch,reward_batch,done_batch) # This is to compute the loss value based on the model's loss function and the data batches + if hasattr(self.nn,'gradient'): # This is a condition to check if the model has a gradient attribute, which stands for a custom gradient function + gradient=self.nn.gradient(tape,loss) # This is to compute the gradients based on the model's gradient function and the tape and loss objects + if hasattr(self.nn.opt,'apply_gradients'): # This is a condition to check if the model's optimizer has an apply_gradients method, which indicates that it is a TensorFlow optimizer + self.nn.opt.apply_gradients(zip(gradient,self.nn.param)) # This is to apply the gradients to the model's parameters using the optimizer's apply_gradients method + else: # This means that the model's optimizer does not have an apply_gradients method, which indicates that it is a custom optimizer function + self.nn.opt(gradient) # This is to update the model's parameters using the optimizer function and the gradients + else: # This means that the model does not have a gradient attribute, which indicates that it uses the default gradient computation + if hasattr(self.nn,'nn'): # This is a condition to check if the model has a nn attribute, which stands for a single neural network model + gradient=tape.gradient(loss,self.nn.param) # This is to compute the gradients based on the tape and loss objects and the model's parameters + self.nn.opt.apply_gradients(zip(gradient,self.nn.param)) # This is to apply the gradients to the model's parameters using the optimizer's apply_gradients method + else: # This means that the model does not have a nn attribute, which indicates that it uses two neural network models, such as actor and critic + actor_gradient=tape.gradient(loss[0],self.nn.param[0]) # This is to compute the gradients for the actor model based on the first element of the loss object and the actor model's parameters + critic_gradient=tape.gradient(loss[1],self.nn.param[1]) # This is to compute the gradients for the critic model based on the second element of the loss object and the critic model's parameters + self.nn.opt.apply_gradients(zip(actor_gradient,self.nn.param[0])) # This is to apply the gradients to the actor model's parameters using the optimizer's apply_gradients method + self.nn.opt.apply_gradients(zip(critic_gradient,self.nn.param[1])) # This is to apply the gradients to the critic model's parameters using the optimizer's apply_gradients method + return loss # This is to return the loss value + + + # This is a method for computing the loss and updating the model parameters using PyTorch + def pytorch_opt(self,state_batch,action_batch,next_state_batch,reward_batch,done_batch): + loss=self.nn.loss(state_batch,action_batch,next_state_batch,reward_batch,done_batch) # This is to compute the loss value based on the model's loss function and the data batches + self.nn.backward(loss) # This is to compute and accumulate the gradients based on the model's backward method and the loss object + self.nn.opt() # This is to update and reset the model's parameters based on the model's opt method, which stands for optimizer function + return loss # This is to return the loss value + + + # This is a method for choosing between tf_opt or pytorch_opt based on the platform attribute + def opt(self,state_batch,action_batch,next_state_batch,reward_batch,done_batch): + if hasattr(self.platform,'DType'): # This is a condition to check if the platform has a DType attribute, which indicates that it is TensorFlow + loss=self.tf_opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch) # This is to call the tf_opt method and pass the data batches as arguments + else: # This means that the platform does not have a DType attribute, which indicates that it is PyTorch + loss=self.pytorch_opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch) # This is to call the pytorch_opt method and pass the data batches as arguments + return loss # This is to return the loss value + + + # This is a method for computing the loss and updating the model parameters using online data + def opt_ol(self,state,action,next_state,reward,done): + if hasattr(self.platform,'DType'): # This is a condition to check if the platform has a DType attribute, which indicates that it is TensorFlow + loss=self.tf_opt(state,action,next_state,reward,done) # This is to call the tf_opt method and pass the online data as arguments + else: # This means that the platform does not have a DType attribute, which indicates that it is PyTorch + loss=self.pytorch_opt(state,action,next_state,reward,done) # This is to call the pytorch_opt method and pass the online data as arguments + return loss # This is to return the loss value + + + # This is a method for storing the data into the pools + def pool(self,s,a,next_s,r,done): + if type(self.state_pool)!=np.ndarray and self.state_pool==None: # This is a condition to check if the state_pool is not an array and is None, which indicates that it is empty + self.state_pool=s # This is to assign the state s to the state_pool + if type(a)==int: # This is a condition to check if the action a is an integer, which indicates that it is a discrete action + a=np.array(a) # This is to convert the action a into an array + self.action_pool=np.expand_dims(a,axis=0) # This is to expand the dimension of the action array and assign it to the action_pool + else: # This means that the action a is not an integer, which indicates that it is a continuous action + self.action_pool=a # This is to assign the action a to the action_pool + self.next_state_pool=np.expand_dims(next_s,axis=0) # This is to expand the dimension of the next state next_s and assign it to the next_state_pool + self.reward_pool=np.expand_dims(r,axis=0) # This is to expand the dimension of the reward r and assign it to the reward_pool + self.done_pool=np.expand_dims(done,axis=0) # This is to expand the dimension of the done flag done and assign it to the done_pool + else: # This means that the state_pool is not empty + self.state_pool=np.concatenate((self.state_pool,s),0) # This is to concatenate or append the state s to the state_pool along axis 0 + if type(a)==int: # This is a condition to check if the action a is an integer, which indicates that it is a discrete action + a=np.array(a) # This is to convert the action a into an array + self.action_pool=np.concatenate((self.action_pool,np.expand_dims(a,axis=0)),0) # This is to expand the dimension of the action array and concatenate or append it to the action_pool along axis 0 + else: # This means that the action a is not an integer, which indicates that it is a continuous action + self.action_pool=np.concatenate((self.action_pool,a),0) # This is to concatenate or append the action a to the action_pool along axis 0 + self.next_state_pool=np.concatenate((self.next_state_pool,np.expand_dims(next_s,axis=0)),0) # This is to expand the dimension of the next state next_s and concatenate or append it to the next_state_pool along axis 0 + self.reward_pool=np.concatenate((self.reward_pool,np.expand_dims(r,axis=0)),0) # This is to expand the dimension of the reward r and concatenate or append it to the reward_pool along axis 0 + self.done_pool=np.concatenate((self.done_pool,np.expand_dims(done,axis=0)),0) # This is to expand the dimension of the done flag done and concatenate or append it to the done_pool along axis 0 + if len(self.state_pool)>self.pool_size: # This is a condition to check if the length of the state_pool exceeds the pool_size attribute, which indicates that it needs trimming + self.state_pool=self.state_pool[1:] # This is to remove or pop out the first element of the state_pool + self.action_pool=self.action_pool[1:] # This is to remove or pop out the first element of the action_pool + self.next_state_pool=self.next_state_pool[1:] # This is to remove or pop out the first element of the next_state_pool + self.reward_pool=self.reward_pool[1:] # This is to remove or pop out the first element of the reward_pool + self.done_pool=self.done_pool[1:] # This is to remove or pop out the first element of the done pool + return + + + # This is a method for training the model using data from pools + def _train(self): + if len(self.state_pool)self.pool_size: # This is a condition to check if the length of the state_pool exceeds the pool_size attribute, which indicates that it needs trimming + TD=np.array(0) # This is to create an array with zero value as TD + self.nn.pr.TD=np.append(TD,self.nn.pr.TD[2:]) # This is to append TD to the TD attribute of pr after removing or popping out the first two elements + self.reward=r+self.reward # This is to update the reward value by adding r to it + loss=self._train() # This is to call the _train method for training the model using data from pools and return the loss value + self.sc+=1 # This is to increase the step counter by one + if done: # This is a condition to check if the done flag is True, which indicates that the episode has ended + if self.save_episode==True: # This is a condition to check if the save_episode attribute is True, which indicates that it needs to save an episode + episode=[s,a,next_s,r] # This is to assign a list of state s, action a, next state next_s and reward r as an episode + self.reward_list.append(self.reward) # This is to append the reward value to the reward_list attribute, which stands for a list of rewards per episode + return loss,episode,done # This is to return the loss value, episode and done flag as outputs + elif self.save_episode==True: # This is a condition to check if the save_episode attribute is True, which indicates that it needs to save an episode + episode=[s,a,next_s,r] # This is to assign a list of state s, action a, next state next_s and reward r as an episode + s=next_s # This is to update the state s by assigning next_s to it + else: # This is an else clause for the case when the episode_step attribute is not None, which indicates that there is a limit on the number of steps per episode + for _ in range(self.episode_step): # This is a loop for running an episode for a fixed number of steps + self.suspend_func() # This is to call the suspend_func method for suspending the training if needed + if hasattr(self.nn,'nn'): # This is a condition to check if the model has a nn attribute, which stands for a single neural network model + if hasattr(self.platform,'DType'): # This is a condition to check if the platform has a DType attribute, which indicates that it is TensorFlow + s=np.expand_dims(s,axis=0) # This is to expand the dimension of the state s along axis 0 + if self.epsilon==None: # This is a condition to check if the epsilon attribute is None, which indicates that it needs to be initialized + self.epsilon=self.nn.epsilon(self.sc) # This is to assign the epsilon value returned by the model's epsilon method, which stands for epsilon function, to the epsilon attribute + action_prob=self.epsilon_greedy_policy(s) # This is to call the epsilon_greedy_policy method for choosing an action based on a state and return its probability + a=np.random.choice(self.action_count,p=action_prob) # This is to sample an action from a discrete distribution with probabilities given by action_prob + else: # This means that the platform does not have a DType attribute, which indicates that it is PyTorch + s=np.expand_dims(s,axis=0) # This is to expand the dimension of the state s along axis 0 + if self.epsilon==None: # This is a condition to check if the epsilon attribute is None, which indicates that it needs to be initialized + self.epsilon=self.nn.epsilon(self.sc) # This is to assign the epsilon value returned by the model's epsilon method, which stands for epsilon function, to the epsilon attribute + action_prob=self.epsilon_greedy_policy(s) # This is to call the epsilon_greedy_policy method for choosing an action based on a state and return its probability + a=np.random.choice(self.action_count,p=action_prob) # This is to sample an action from a discrete distribution with probabilities given by action_prob + else: # This means that the model does not have a nn attribute, which indicates that it uses two neural network models, such as actor and critic + if hasattr(self.platform,'DType'): # This is a condition to check if the platform has a DType attribute, which indicates that it is TensorFlow + s=np.expand_dims(s,axis=0) # This is to expand the dimension of the state s along axis 0 + if self.epsilon==None: # This is a condition to check if the epsilon attribute is None, which indicates that it needs to be initialized + self.epsilon=self.nn.epsilon(self.sc) # This is to assign the epsilon value returned by the model's epsilon method, which stands for epsilon function, to the epsilon attribute + if hasattr(self.nn,'discriminator'): # This is a condition to check if the model has a discriminator attribute, which stands for a discriminator model that evaluates the quality of an action + a=self.nn.action(s) # This is to get an action from the model's action method, which stands for action function, based on the state s + reward=self.nn.discriminator(s,a) # This is to get a reward from the model's discriminator method, which stands for discriminator function, based on the state s and action a + s=np.squeeze(s) # This is to remove any singleton dimensions from the state s + else: # This means that the model does not have a discriminator attribute, which indicates that it uses the environment's reward function + a=self.nn.action(s).numpy() # This is to get an action from the model's action method, which stands for action function, based on the state s and convert it into a numpy array + else: # This means that the platform does not have a DType attribute, which indicates that it is PyTorch + s=np.expand_dims(s,axis=0) # This is to expand the dimension of the state s along axis 0 + if self.epsilon==None: # This is a condition to check if the epsilon attribute is None, which indicates that it needs to be initialized + self.epsilon=self.nn.epsilon(self.sc) # This is to assign the epsilon value returned by the model's epsilon method, which stands for epsilon function, to the epsilon attribute + if hasattr(self.nn,'discriminator'): # This is a condition to check if the model has a discriminator attribute, which stands for a discriminator model that evaluates the quality of an action + a=self.nn.action(s) # This is to get an action from the model's action method, which stands for action function, based on the state s + reward=self.nn.discriminator(s,a) # This is to get a reward from the model's discriminator method, which stands for discriminator function, based on the state s and action a + s=np.squeeze(s) # This is to remove any singleton dimensions from the state s + else: # This means that the model does not have a discriminator attribute, which indicates that it uses the environment's reward function + a=self.nn.action(s).detach().numpy() # This is to get an action from the model's action method, which stands for action function, based on the state s and detach it from the computation graph and convert it into a numpy array + next_s,r,done=self.nn.env(a) # This is to get the next state next_s, reward r and done flag done from the environment using the model's env method based on the action a + if hasattr(self.platform,'DType'): # This is a condition to check if the platform has a DType attribute, which indicates that it is TensorFlow + if type(self.nn.param[0])!=list: # This is a condition to check if the first element of the model's param attribute, which stands for parameters, is not a list, which indicates that it has only one neural network model + next_s=np.array(next_s,self.nn.param[0].dtype.name) # This is to convert the next state next_s into an array with data type matching with the model's parameter data type + r=np.array(r,self.nn.param[0].dtype.name) # This is to convert the reward r into an array with data type matching with the model's parameter data type + done=np.array(done,self.nn.param[0].dtype.name) # This is to convert the done flag done into an array with data type matching with the model's parameter data type + else: # This means that the first element of the model's param attribute is a list, which indicates that it has two neural network models, such as actor and critic + next_s=np.array(next_s,self.nn.param[0][0].dtype.name) # This is to convert the next state next_s into an array with data type matching with the first model’s parameter data type + r=np.array(r,self.nn.param[0][0].dtype.name) # This is to convert the reward r into an array with data type matching with the first model's parameter data type + done=np.array(done,self.nn.param[0][0].dtype.name) # This is to convert the done flag done into an array with data type matching with the first model's parameter data type + if hasattr(self.nn,'pool'): # This is a condition to check if the model has a pool attribute, which stands for a custom pool function + if hasattr(self.nn,'discriminator'): # This is a condition to check if the model has a discriminator attribute, which stands for a discriminator model that evaluates the quality of an action + self.nn.pool(self.state_pool,self.action_pool,self.next_state_pool,self.reward_pool,self.done_pool,[s,a,next_s,reward,done]) # This is to call the model's pool method for storing the data into the pools using the state s, action a, next state next_s, reward reward and done flag done + else: # This means that the model does not have a discriminator attribute, which indicates that it uses the environment's reward function + self.nn.pool(self.state_pool,self.action_pool,self.next_state_pool,self.reward_pool,self.done_pool,[s,a,next_s,r,done]) # This is to call the model's pool method for storing the data into the pools using the state s, action a, next state next_s, reward r and done flag done + else: # This means that the model does not have a pool attribute, which indicates that it uses the default pool method + self.pool(s,a,next_s,r,done) # This is to call the self.pool method for storing the data into the pools using the state s, action a, next state next_s, reward r and done flag done + if hasattr(self.nn,'pr'): # This is a condition to check if the model has a pr attribute, which stands for priority replay + self.nn.pr.TD=np.append(self.nn.pr.TD,self.nn.initial_TD) # This is to append the initial TD value returned by the model's initial_TD method, which stands for initial temporal difference function, to the TD attribute of pr, which stands for temporal difference values + if len(self.state_pool)>self.pool_size: # This is a condition to check if the length of the state_pool exceeds the pool_size attribute, which indicates that it needs trimming + TD=np.array(0) # This is to create an array with zero value as TD + self.nn.pr.TD=np.append(TD,self.nn.pr.TD[2:]) # This is to append TD to the TD attribute of pr after removing or popping out the first two elements + self.reward=r+self.reward # This is to update the reward value by adding r to it + loss=self._train() # This is to call the _train method for training the model using data from pools and return the loss value + self.sc+=1 # This is to increase the step counter by one + if done: # This is a condition to check if the done flag is True, which indicates that the episode has ended + if self.save_episode==True: # This is a condition to check if the save_episode attribute is True, which indicates that it needs to save an episode + episode=[s,a,next_s,r] # This is to assign a list of state s, action a, next state next_s and reward r as an episode + self.reward_list.append(self.reward) # This is to append the reward value to the reward_list attribute, which stands for a list of rewards per episode + return loss,episode,done # This is to return the loss value, episode and done flag as outputs + elif self.save_episode==True: # This is a condition to check if the save_episode attribute is True, which indicates that it needs to save an episode + episode=[s,a,next_s,r] # This is to assign a list of state s, action a, next state next_s and reward r as an episode + s=next_s # This is to update the state s by assigning next_s to it + self.reward_list.append(self.reward) # This is to append the reward value to the reward_list attribute, which stands for a list of rewards per episode + return loss,episode,done # This is to return the loss value, episode and done flag as outputs + + + # This is a function for training the model using data from episodes or pools + def train(self,episode_count,save=None,one=True,p=None,s=None): + avg_reward=None # This is to initialize the average reward value to None + if p==None: # This is a condition to check if the p parameter is None, which indicates that it needs to be assigned a default value + self.p=9 # This is to assign 9 to the p attribute, which stands for the frequency of printing the loss and reward values + else: # This means that the p parameter is not None, which indicates that it has a given value + self.p=p-1 # This is to assign p-1 to the p attribute, which stands for the frequency of printing the loss and reward values + if s==None: # This is a condition to check if the s parameter is None, which indicates that it needs to be assigned a default value + self.s=1 # This is to assign 1 to the s attribute, which stands for the frequency of saving the model parameters + self.file_list=None # This is to assign None to the file_list attribute, which stands for a list of file names for saving the model parameters + else: # This means that the s parameter is not None, which indicates that it has a given value + self.s=s-1 # This is to assign s-1 to the s attribute, which stands for the frequency of saving the model parameters + self.file_list=[] # This is to initialize an empty list for the file_list attribute, which stands for a list of file names for saving the model parameters + if episode_count!=None: # This is a condition to check if the episode_count parameter is not None, which indicates that it has a given value + for i in range(episode_count): # This is a loop for running a given number of episodes + t1=time.time() # This is to record the start time of an episode + loss,episode,done=self.train_() # This is to call the train_ method for training the model using data from an episode and return the loss value, episode and done flag + if self.trial_count!=None: # This is a condition to check if the trial_count attribute is not None, which indicates that it has a defined value + if len(self.reward_list)>=self.trial_count: # This is a condition to check if the length of the reward_list attribute, which stands for a list of rewards per episode, is greater than or equal to the trial_count attribute, which stands for the number of episodes for calculating the average reward + avg_reward=statistics.mean(self.reward_list[-self.trial_count:]) # This is to calculate the average reward value by taking the mean of the last trial_count elements of the reward_list attribute + if self.criterion!=None and avg_reward>=self.criterion: # This is a condition to check if the criterion attribute is not None and the average reward value is greater than or equal to the criterion attribute, which indicates that the model has reached a desired performance level + t2=time.time() # This is to record the end time of an episode + self.total_time+=(t2-t1) # This is to update the total_time attribute, which stands for the total time spent on training, by adding (t2-t1), which stands for the time spent on an episode + time_=self.total_time-int(self.total_time) # This is to calculate the decimal part of the total_time attribute + if time_<0.5: # This is a condition to check if the decimal part of the total_time attribute is less than 0.5, which indicates that it needs rounding down + self.total_time=int(self.total_time) # This is to round down the total_time attribute by converting it into an integer + else: # This means that the decimal part of the total_time attribute is greater than or equal to 0.5, which indicates that it needs rounding up + self.total_time=int(self.total_time)+1 # This is to round up + print('episode:{0}'.format(self.total_episode)) # This is to print the episode number + print('last loss:{0:.6f}'.format(loss)) # This is to print the last loss value with six decimal places + print('average reward:{0}'.format(avg_reward)) # This is to print the average reward value + print() # This is to print an empty line for spacing + print('time:{0}s'.format(self.total_time)) # This is to print the total time spent on training in seconds + return # This is to return from the function + self.loss=loss # This is to assign the loss value to the loss attribute + self.loss_list.append(loss) # This is to append the loss value to the loss_list attribute, which stands for a list of losses per episode + self.total_episode+=1 # This is to increase the total_episode attribute, which stands for the total number of episodes, by one + if episode_count%10!=0: # This is a condition to check if the episode_count parameter is not divisible by 10, which indicates that it needs some adjustment for calculating p and s + p=episode_count-episode_count%self.p # This is to subtract the remainder of dividing the episode_count parameter by the p attribute from the episode_count parameter + p=int(p/self.p) # This is to divide p by the p attribute and convert it into an integer + s=episode_count-episode_count%self.s # This is to subtract the remainder of dividing the episode_count parameter by the s attribute from the episode_count parameter + s=int(s/self.s) # This is to divide s by the s attribute and convert it into an integer + else: # This means that the episode_count parameter is divisible by 10, which indicates that it does not need any adjustment for calculating p and s + p=episode_count/(self.p+1) # This is to divide the episode_count parameter by (the p attribute plus one) + p=int(p) # This is to convert p into an integer + s=episode_count/(self.s+1) # This is to divide the episode_count parameter by (the s attribute plus one) + s=int(s) # This is to convert s into an integer + if p==0: # This is a condition to check if p is zero, which indicates that it needs to be assigned a minimum value of one + p=1 # This is to assign one to p + if s==0: # This is a condition to check if s is zero, which indicates that it needs to be assigned a minimum value of one + s=1 # This is to assign one to s + if i%p==0: # This is a condition to check if i, which stands for the episode index, is divisible by p, which indicates that it is time to print the loss and reward values + if len(self.state_pool)>=self.batch: # This is a condition to check if the length of the state_pool attribute, which stands for a pool of states, is greater than or equal to the batch attribute, which stands for a batch size for training + print('episode:{0} loss:{1:.6f}'.format(i+1,loss)) # This is to print the episode number and the loss value with six decimal places + if avg_reward!=None: # This is a condition to check if the average reward value is not None, which indicates that it has been calculated + print('episode:{0} average reward:{1}'.format(i+1,avg_reward)) # This is to print the episode number and the average reward value + else: # This means that the average reward value is None, which indicates that it has not been calculated yet + print('episode:{0} reward:{1}'.format(i+1,self.reward)) # This is to print the episode number and the reward value + print() # This is to print an empty line for spacing + if save!=None and i%s==0: # This is a condition to check if the save parameter is not None and i, which stands for the episode index, is divisible by s, which indicates that it is time to save the model parameters + self.save(self.total_episode,one) # This is to call the save method for saving the model parameters using the total_episode attribute as an identifier and one as a flag for indicating whether to overwrite or append files + if self.save_episode==True: # This is a condition to check if the save_episode attribute is True, which indicates that it needs to save an episode + if done: # This is a condition to check if done flag + episode.append('done') + self.episode_set.append(episode) # This is to append the episode list to the episode_set attribute, which stands for a set of episodes + if self.max_episode_count!=None and len(self.episode_set)>=self.max_episode_count: # This is a condition to check if the max_episode_count attribute is not None and the length of the episode_set attribute is greater than or equal to the max_episode_count attribute, which indicates that it has reached the maximum number of episodes to save + self.save_episode=False # This is to assign False to the save_episode attribute, which indicates that it does not need to save any more episodes + try: # This is a try block for handling any possible errors + try: # This is a nested try block for handling any possible errors + self.nn.ec.assign_add(1) # This is to call the assign_add method of the ec attribute of the model, which stands for episode counter, to increase it by one + except Exception: # This is an except block for catching any exceptions raised by the nested try block + self.nn.ec+=1 # This is to increase the ec attribute of the model by one using the += operator + except Exception: # This is an except block for catching any exceptions raised by the outer try block + pass # This is a pass statement for doing nothing and continuing the execution + t2=time.time() # This is to record the end time of an episode + self.time+=(t2-t1) # This is to update the time attribute, which stands for the time spent on an episode, by adding (t2-t1), which stands for the time difference between t2 and t1 + else: # This is an else clause for the condition of the episode_count parameter, which indicates that it does not have a given value + i=0 # This is to initialize the episode index to zero + while True: # This is an infinite loop for running episodes until the model reaches the desired performance level + t1=time.time() # This is to record the start time of an episode + loss,episode,done=self.train_() # This is to call the train_ method for training the model using data from an episode and return the loss value, episode and done flag + if self.trial_count!=None: # This is a condition to check if the trial_count attribute is not None, which indicates that it has a defined value + if len(self.reward_list)==self.trial_count: # This is a condition to check if the length of the reward_list attribute, which stands for a list of rewards per episode, is equal to the trial_count attribute, which stands for the number of episodes for calculating the average reward + avg_reward=statistics.mean(self.reward_list[-self.trial_count:]) # This is to calculate the average reward value by taking the mean of the last trial_count elements of the reward_list attribute + if self.criterion!=None and avg_reward>=self.criterion: # This is a condition to check if the criterion attribute is not None and the average reward value is greater than or equal to the criterion attribute, which indicates that the model has reached a desired performance level + t2=time.time() # This is to record the end time of an episode + self.total_time+=(t2-t1) # This is to update the total_time attribute, which stands for the total time spent on training, by adding (t2-t1), which stands for the time spent on an episode + time_=self.total_time-int(self.total_time) # This is to calculate the decimal part of the total_time attribute + if time_<0.5: # This is a condition to check if the decimal part of the total_time attribute is less than 0.5, which indicates that it needs rounding down + self.total_time=int(self.total_time) # This is to round down the total_time attribute by converting it into an integer + else: # This means that the decimal part of the total_time attribute is greater than or equal to 0.5, which indicates that it needs rounding up + self.total_time=int(self.total_time)+1 # This is to round up + print('episode:{0}'.format(self.total_episode)) # This is to print the episode number + print('last loss:{0:.6f}'.format(loss)) # This is to print the last loss value with six decimal places + print('average reward:{0}'.format(avg_reward)) # This is to print the average reward value + print() # This is to print an empty line for spacing + print('time:{0}s'.format(self.total_time)) # This is to print the total time spent on training in seconds + return # This is to return from the function + self.loss=loss # This is to assign the loss value to the loss attribute + self.loss_list.append(loss) # This is to append the loss value to the loss_list attribute, which stands for a list of losses per episode + self.total_episode+=1 # This is to increase the total_episode attribute, which stands for the total number of episodes, by one + if episode_count%10!=0: # This is a condition to check if the episode_count parameter is not divisible by 10, which indicates that it needs some adjustment for calculating p and s + p=episode_count-episode_count%self.p # This is to subtract the remainder of dividing the episode_count parameter by the p attribute from the episode_count parameter + p=int(p/self.p) # This is to divide p by the p attribute and convert it into an integer + s=episode_count-episode_count%self.s # This is to subtract the remainder of dividing the episode_count parameter by the s attribute from the episode_count parameter + s=int(s/self.s) # This is to divide s by the s attribute and convert it into an integer + else: # This means that the episode_count parameter is divisible by 10, which indicates that it does not need any adjustment for calculating p and s + p=episode_count/(self.p+1) # This is to divide the episode_count parameter by (the p attribute plus one) + p=int(p) # This is to convert p into an integer + s=episode_count/(self.s+1) # This is to divide the episode_count parameter by (the s attribute plus one) + s=int(s) # This is to convert s into an integer + if p==0: # This is a condition to check if p is zero, which indicates that it needs to be assigned a minimum value of one + p=1 # This is to assign one to p + if s==0: # This is a condition to check if s is zero, which indicates that it needs to be assigned a minimum value of one + s=1 # This is to assign one to s + if i%p==0: # This is a condition to check if i, which stands for the episode index, is divisible by p, which indicates that it is time to print the loss and reward values + if len(self.state_pool)>=self.batch: # This is a condition to check if the length of the state_pool attribute, which stands for a pool of states, is greater than or equal to the batch attribute, which stands for a batch size for training + print('episode:{0} loss:{1:.6f}'.format(i+1,loss)) # This is to print the episode number and the loss value with six decimal places + if avg_reward!=None: # This is a condition to check if the average reward value is not None, which indicates that it has been calculated + print('episode:{0} average reward:{1}'.format(i+1,avg_reward)) # This is to print the episode number and the average reward value + else: # This means that the average reward value is None, which indicates that it has not been calculated yet + print('episode:{0} reward:{1}'.format(i+1,self.reward)) # This is to print the episode number and the reward value + print() # This is to print an empty line for spacing + if save!=None and i%s==0: # This is a condition to check if the save parameter is not None and i, which stands for the episode index, is divisible by s, which indicates that it is time to save the model parameters + self.save(self.total_episode,one) # This is to call the save method for saving the model parameters using the total_episode attribute as an identifier and one as a flag for indicating whether to overwrite or append files + if self.save_episode==True: # This is a condition to check if the save_episode attribute is True, which indicates that it needs to save an episode + if done: # This is a condition to check if done flag + episode.append('done') + self.episode_set.append(episode) # This is to append the episode list to the episode_set attribute, which stands for a set of episodes + if self.max_episode_count!=None and len(self.episode_set)>=self.max_episode_count: # This is a condition to check if the max_episode_count attribute is not None and the length of the episode_set attribute is greater than or equal to the max_episode_count attribute, which indicates that it has reached the maximum number of episodes to save + self.save_episode=False # This is to assign False to the save_episode attribute, which indicates that it does not need to save any more episodes + try: # This is a try block for handling any possible errors + try: # This is a nested try block for handling any possible errors + self.nn.ec.assign_add(1) # This is to call the assign_add method of the ec attribute of the model, which stands for episode counter, to increase it by one + except Exception: # This is an except block for catching any exceptions raised by the nested try block + self.nn.ec+=1 # This is to increase the ec attribute of the model by one using the += operator + except Exception: # This is an except block for catching any exceptions raised by the outer try block + pass # This is a pass statement for doing nothing and continuing the execution + t2=time.time() # This is to record the end time of an episode + self.time+=(t2-t1) # This is to update the time attribute, which stands for the time spent on an episode, by adding (t2-t1), which stands for the time difference between t2 and t1 + time_=self.time-int(self.time) + time_=self.time-int(self.time) # This is to calculate the decimal part of the time attribute, which stands for the time spent on an episode + if time_<0.5: # This is a condition to check if the decimal part of the time attribute is less than 0.5, which indicates that it needs rounding down + self.total_time=int(self.time) # This is to round down the time attribute by converting it into an integer and assign it to the total_time attribute, which stands for the total time spent on training + else: # This means that the decimal part of the time attribute is greater than or equal to 0.5, which indicates that it needs rounding up + self.total_time=int(self.time)+1 # This is to round up the time attribute by converting it into an integer and adding one and assign it to the total_time attribute + self.total_time+=self.time # This is to update the total_time attribute by adding the time attribute to it + print('last loss:{0:.6f}'.format(loss)) # This is to print the last loss value with six decimal places + print('last reward:{0}'.format(self.reward)) # This is to print the last reward value + print() # This is to print an empty line for spacing + print('time:{0}s'.format(self.time)) # This is to print the time spent on an episode in seconds + return + + + # This is a function for training the model online using data from the environment + def train_online(self): + while True: # This is an infinite loop for running online training until the model stops or suspends + if hasattr(self.nn,'save'): # This is a condition to check if the model has a save attribute, which stands for a save function + self.nn.save(self.save) # This is to call the model's save method for saving the model parameters using the save attribute, which stands for a file name + if hasattr(self.nn,'stop_flag'): # This is a condition to check if the model has a stop_flag attribute, which stands for a flag for stopping the training + if self.nn.stop_flag==True: # This is a condition to check if the stop_flag attribute is True, which indicates that the training needs to stop + return # This is to return from the function + if hasattr(self.nn,'stop_func'): # This is a condition to check if the model has a stop_func attribute, which stands for a stop function + if self.nn.stop_func(): # This is to call the model's stop_func method and check if it returns True, which indicates that the training needs to stop + return # This is to return from the function + if hasattr(self.nn,'suspend_func'): # This is a condition to check if the model has a suspend_func attribute, which stands for a suspend function + self.nn.suspend_func() # This is to call the model's suspend_func method for suspending the training if needed + data=self.nn.online() # This is to call the model's online method for getting data from the environment and assign it to data + if data=='stop': # This is a condition to check if data is 'stop', which indicates that the training needs to stop + return # This is to return from the function + elif data=='suspend': # This is a condition to check if data is 'suspend', which indicates that the training needs to suspend + self.nn.suspend_func() # This is to call the model's suspend_func method for suspending the training + loss=self.opt_ol(data[0],data[1],data[2],data[3],data[4]) # This is to call the opt_ol method for optimizing the model using data and return the loss value. The data consists of five elements: state, action, next state, reward and done flag. + loss=loss.numpy() # This is to convert the loss value into a numpy array + self.nn.train_loss_list.append(loss) # This is to append the loss value to the train_loss_list attribute of the model, which stands for a list of losses per step + if len(self.nn.train_acc_list)==self.nn.max_length: # This is a condition to check if the length of the train_acc_list attribute of the model, which stands for a list of accuracies per step, is equal to the max_length attribute of the model, which stands for the maximum length of lists + del self.nn.train_acc_list[0] # This is to delete or pop out the first element of the train_acc_list attribute of the model, which stands for removing the oldest accuracy value + if hasattr(self.nn,'counter'): # This is a condition to check if the model has a counter attribute, which stands for a counter for steps or epochs + self.nn.counter+=1 # This is to increase the counter attribute + return \ No newline at end of file From dc854d271074e35f9a7602d0f7002348134a1eaa Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Wed, 26 Jul 2023 13:22:17 +0800 Subject: [PATCH 090/337] Update README.md --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 6dfb015b..94676fde 100644 --- a/README.md +++ b/README.md @@ -78,7 +78,7 @@ kernel.test(x_test,y_test,32)#test the network performance on the test set with ``` ## Parallel test: -**You can download neural network example in this link,and then you can import neural network and train with kernel,link and example code are below.** +**You can get neural network example from the link below, and then you can import neural network and train with kernel, example code are below.** https://github.com/NoteDancing/Note-documentation/blob/Note-7.0/Note%207.0%20documentation/DL/neural%20network/tensorflow/process/nn.py @@ -316,7 +316,7 @@ for p in range(7): #loop over the processes ## RL： **Pool Network:** -**You can download neural network example in this link,and then you can import neural network and train with kernel,link and example code are below.** +**You can get neural network example from the link below, and then you can import neural network and train with kernel, example code are below.** https://github.com/NoteDancing/Note-documentation/blob/Note-7.0/Note%207.0%20documentation/RL/neural%20network/tensorflow/pool%20network/DQN.py @@ -375,7 +375,7 @@ for p in range(5): #loop over the processes # Parallel test: -**You can download neural network example in this link,and then you can import neural network and train with kernel,link and example code are below.** +**You can get neural network example from the link below, and then you can import neural network and train with kernel, example code are below.** https://github.com/NoteDancing/Note-documentation/blob/Note-7.0/Note%207.0%20documentation/DL/neural%20network/tensorflow/process/nn.py @@ -398,7 +398,7 @@ loss=test.loss_acc() #calculate the loss and accuracy of the test # Online training: -**You can download neural network example in this link,and then you can import neural network and train with kernel,link and example code are below.** +**You can get neural network example from the link below, and then you can import neural network and train with kernel, example code are below.** https://github.com/NoteDancing/Note-documentation/blob/Note-7.0/Note%207.0%20documentation/DL/neural%20network/tensorflow/nn_ol.py From 755a0b1476c94c1facfaaa96e89096f49bb82b24 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 27 Jul 2023 13:08:13 +0800 Subject: [PATCH 091/337] Update kernel.py --- .../DL/kernel/process/kernel.py | 43 ++++++------------- 1 file changed, 14 insertions(+), 29 deletions(-) diff --git a/Note 7.0 documentation/DL/kernel/process/kernel.py b/Note 7.0 documentation/DL/kernel/process/kernel.py index 265c7da3..f3e68f45 100644 --- a/Note 7.0 documentation/DL/kernel/process/kernel.py +++ b/Note 7.0 documentation/DL/kernel/process/kernel.py @@ -21,36 +21,21 @@ def __init__(self,nn=None): self.batches=None # the number of batches per epoch self.buffer_size=None # the buffer size for shuffling the data self.priority_flag=False # a flag to indicate whether to use priority optimization - self.priority_p=0 # the priority process index self.max_opt=None # the maximum number of optimization steps for each process self.epoch=None # the number of epochs for training - self.epoch_counter=0 # the counter for epochs self.stop=False # a flag to indicate whether to stop the training - self.stop_flag=False # a flag to indicate whether to stop the training by external signal - self.save_flag=False # a flag to indicate whether to save the model parameters self.save_epoch=None # the epoch interval for saving the model parameters self.batch=None # the batch size for training and testing data - self.epoch_=0 # a counter for epochs in online mode self.end_loss=None # the end condition for training loss self.end_acc=None # the end condition for training accuracy self.end_test_loss=None # the end condition for testing loss self.end_test_acc=None # the end condition for testing accuracy self.acc_flag='%' # a flag to indicate whether to use percentage or decimal for accuracy display - self.opt_counter=None # a counter for optimization steps for each process self.p=None # the process index for online mode self.s=None # a signal object for online mode self.saving_one=True # a flag to indicate whether to save only one copy of model parameters or multiple copies with different names self.filename='save.dat' # the default filename for saving model parameters - self.train_loss=0 # the current training loss value - self.train_acc=0 # the current training accuracy value - self.train_loss_list=[] # the list of training loss values over epochs - self.train_acc_list=[] # the list of training accuracy values over epochs - self.test_loss=0 # the current testing loss value - self.test_acc=0 # the current testing accuracy value - self.test_loss_list=[] # the list of testing loss values over epochs - self.test_acc_list=[] # the list of testing accuracy values over epochs self.test_flag=False # a flag to indicate whether to use testing data or not - self.total_epoch=0 # the total number of epochs for training def data(self,train_data=None,train_labels=None,test_data=None,test_labels=None,train_dataset=None,test_dataset=None): @@ -113,24 +98,24 @@ def segment_data(self): def init(self,manager): # a method to initialize some shared variables for multiprocessing - self.epoch_counter=Value('i',self.epoch_counter) # create a shared value for epoch counter + self.epoch_counter=Value('i',0) # create a shared value for epoch counter self.batch_counter=Array('i',self.batch_counter) # create a shared array for batch counter self.total_loss=Array('f',self.total_loss) # create a shared array for total loss - self.total_epoch=Value('i',self.total_epoch) # create a shared value for total epoch - self.train_loss=Value('f',self.train_loss) # create a shared value for training loss - self.train_loss_list=manager.list(self.train_loss_list) # create a shared list for training loss list - self.priority_p=Value('i',self.priority_p) # create a shared value for priority process index + self.total_epoch=Value('i',0) # create a shared value for total epoch + self.train_loss=Value('f',0) # create a shared value for training loss + self.train_loss_list=manager.list([]) # create a shared list for training loss list + self.priority_p=Value('i',0) # create a shared value for priority process index if self.test_flag==True: - self.test_loss=Value('f',self.test_loss) # create a shared value for testing loss if using testing data or dataset - self.test_loss_list=manager.list(self.test_loss_list) # create a shared list for testing loss list if using testing data or dataset + self.test_loss=Value('f',0) # create a shared value for testing loss if using testing data or dataset + self.test_loss_list=manager.list([]) # create a shared list for testing loss list if using testing data or dataset if hasattr(self.nn,'accuracy'): if self.nn.accuracy!=None: self.total_acc=Array('f',self.total_acc) # create a shared array for total accuracy if using accuracy metric - self.train_acc=Value('f',self.train_acc) # create a shared value for training accuracy if using accuracy metric - self.train_acc_list=manager.list(self.train_acc_list) # create a shared list for training accuracy list if using accuracy metric + self.train_acc=Value('f',0) # create a shared value for training accuracy if using accuracy metric + self.train_acc_list=manager.list([]) # create a shared list for training accuracy list if using accuracy metric if self.test_flag==True: - self.test_acc=Value('f',self.test_acc) # create a shared value for testing accuracy if using testing data or dataset and accuracy metric - self.test_acc_list=manager.list(self.test_acc_list) # create a shared list for testing accuracy list if using testing data or dataset and accuracy metric + self.test_acc=Value('f',0) # create a shared value for testing accuracy if using testing data or dataset and accuracy metric + self.test_acc_list=manager.list([]) # create a shared list for testing accuracy list if using testing data or dataset and accuracy metric if self.priority_flag==True: self.opt_counter=Array('i',self.opt_counter) # create a shared array for optimization counter if using priority optimization try: @@ -145,9 +130,9 @@ def init(self,manager): self.nn.bc=manager.list([self.nn.bc]) # create a shared list for batch counter in the neural network model except Exception: self.bc_=manager.list() # create an empty list if there is no batch counter in the neural network model - self.epoch_=Value('i',self.epoch_) # create a shared value for epoch counter in online mode - self.stop_flag=Value('b',self.stop_flag) # create a shared value for stop flag by external signal - self.save_flag=Value('b',self.save_flag) # create a shared value for save flag by external signal + self.epoch_=Value('i',0) # create a shared value for epoch counter in online mode + self.stop_flag=Value('b',False) # create a shared value for stop flag by external signal + self.save_flag=Value('b',False) # create a shared value for save flag by external signal self.file_list=manager.list([]) # create an empty list for file names of saved model parameters self.param=manager.dict() # create an empty dictionary for model parameters self.param[7]=self.nn.param # set the 7th key of the dictionary to be the model parameters of the neural network model From 3418c88c777fe1e1c240e18c5d96f7f89ccc52f6 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 27 Jul 2023 13:10:34 +0800 Subject: [PATCH 092/337] Update kernel_pytorch.py --- .../DL/kernel/process/kernel_pytorch.py | 45 +++++++------------ 1 file changed, 15 insertions(+), 30 deletions(-) diff --git a/Note 7.0 documentation/DL/kernel/process/kernel_pytorch.py b/Note 7.0 documentation/DL/kernel/process/kernel_pytorch.py index 699e53fe..45939718 100644 --- a/Note 7.0 documentation/DL/kernel/process/kernel_pytorch.py +++ b/Note 7.0 documentation/DL/kernel/process/kernel_pytorch.py @@ -14,36 +14,21 @@ def __init__(self,nn=None): # define the constructor method of the class self.batches_t=None # initialize the batches_t attribute to None self.shuffle=False # initialize the shuffle attribute to False self.priority_flag=False # initialize the priority_flag attribute to False - self.priority_p=0 # initialize the priority_p attribute to 0 self.max_opt=None # initialize the max_opt attribute to None self.epoch=None # initialize the epoch attribute to None - self.epoch_counter=0 # initialize the epoch_counter attribute to 0 self.stop=False # initialize the stop attribute to False - self.stop_flag=False # initialize the stop_flag attribute to False - self.save_flag=False # initialize the save_flag attribute to False self.save_epoch=None # initialize the save_epoch attribute to None self.batch=None # initialize the batch attribute to None - self.epoch_=0 # initialize the epoch_ attribute to 0 self.end_loss=None # initialize the end_loss attribute to None self.end_acc=None # initialize the end_acc attribute to None self.end_test_loss=None # initialize the end_test_loss attribute to None self.end_test_acc=None # initialize the end_test_acc attribute to None self.acc_flag='%' # initialize the acc_flag attribute to '%' - self.opt_counter=None # initialize the opt_counter attribute to None self.p=None # initialize the p attribute to None self.s=None # initialize the s attribute to None self.saving_one=True # initialize the saving_one attribute to True self.filename='save.dat' # initialize the filename attribute to 'save.dat' - self.train_loss=0 # initialize the train_loss attribute to 0 - self.train_acc=0 # initialize the train_acc attribute to 0 - self.train_loss_list=[] # initialize the train_loss_list attribute to an empty list - self.train_acc_list=[] # initialize the train_acc_list attribute to an empty list - self.test_loss=0 # initialize the test_loss attribute to 0 - self.test_acc=0 # initialize the test_acc attribute to 0 - self.test_loss_list=[] # initialize the test_loss_list attribute to an empty list - self.test_acc_list=[] # initialize the test_acc_list attribute to an empty list self.test_flag=False # initialize the test_flag attribute to False - self.total_epoch=0 # initialize the total_epoch attribute to 0 def data(self,train_dataset=None,test_dataset=None): @@ -85,23 +70,23 @@ def init(self,manager): '''The manager argument is expected to be a multiprocessing.Manager object that can create shared variables across processes''' - self.epoch_counter=Value('i',self.epoch_counter) # create a shared variable with an integer data type and an initial value equal to the epoch_counter attribute, and assign it to the epoch_counter attribute + self.epoch_counter=Value('i',0) # create a shared variable with an integer data type and an initial value equal to the epoch_counter attribute, and assign it to the epoch_counter attribute self.batch_counter=Array('i',self.batch_counter) # create a shared array with an integer data type and an initial value equal to the batch_counter attribute, and assign it to the batch_counter attribute self.total_loss=Array('f',self.total_loss) # create a shared array with a float data type and an initial value equal to the total_loss attribute, and assign it to the total_loss attribute - self.total_epoch=Value('i',self.total_epoch) # create a shared variable with an integer data type and an initial value equal to the total_epoch attribute, and assign it to the total_epoch attribute - self.train_loss=Value('f',self.train_loss) # create a shared variable with a float data type and an initial value equal to the train_loss attribute, and assign it to the train_loss attribute - self.train_loss_list=manager.list(self.train_loss_list) # create a shared list with an initial value equal to the train_loss_list attribute, using the list method of the manager argument, and assign it to the train_loss_list attribute - self.priority_p=Value('i',self.priority_p) # create a shared variable with an integer data type and an initial value equal to the priority_p attribute, and assign it to the priority_p attribute + self.total_epoch=Value('i',0) # create a shared variable with an integer data type and an initial value equal to the total_epoch attribute, and assign it to the total_epoch attribute + self.train_loss=Value('f',0) # create a shared variable with a float data type and an initial value equal to the train_loss attribute, and assign it to the train_loss attribute + self.train_loss_list=manager.list([]) # create a shared list with an initial value equal to the train_loss_list attribute, using the list method of the manager argument, and assign it to the train_loss_list attribute + self.priority_p=Value('i',0) # create a shared variable with an integer data type and an initial value equal to the priority_p attribute, and assign it to the priority_p attribute if self.test_flag==True: # check if the test_flag attribute is True - self.test_loss=Value('f',self.test_loss) # create a shared variable with a float data type and an initial value equal to the test_loss attribute, and assign it to the test_loss attribute - self.test_loss_list=manager.list(self.test_loss_list) # create a shared list with an initial value equal to the test_loss_list attribute, using the list method of the manager argument, and assign it to the test_loss_list attribute + self.test_loss=Value('f',0) # create a shared variable with a float data type and an initial value equal to the test_loss attribute, and assign it to the test_loss attribute + self.test_loss_list=manager.list([]) # create a shared list with an initial value equal to the test_loss_list attribute, using the list method of the manager argument, and assign it to the test_loss_list attribute if hasattr(self.nn,'accuracy'): # check if the nn attribute has a method named accuracy self.total_acc=Array('f',self.total_acc) # create a shared array with a float data type and an initial value equal to the total_acc attribute, and assign it to the total_acc attribute - self.train_acc=Value('f',self.train_acc) # create a shared variable with a float data type and an initial value equal to the train_acc attribute, and assign it to the train_acc attribute - self.train_acc_list=manager.list(self.train_acc_list) # create a shared list with an initial value equal to the train_acc_list attribute, using the list method of the manager argument, and assign it to the train_acc_list attribute + self.train_acc=Value('f',0) # create a shared variable with a float data type and an initial value equal to the train_acc attribute, and assign it to the train_acc attribute + self.train_acc_list=manager.list([]) # create a shared list with an initial value equal to the train_acc_list attribute, using the list method of the manager argument, and assign it to the train_acc_list attribute if self.test_flag==True: # check if the test_flag attribute is True - self.test_acc=Value('f',self.test_acc) # create a shared variable with a float data type and an initial value equal to the test_acc attribute, and assign it to the test_acc attribute - self.test_acc_list=manager.list(self.test_acc_list) # create a shared list with an initial value equal to the test_acc_list attribute, using the list method of the manager argument, and assign it to the test_acc_list attribute + self.test_acc=Value('f',0) # create a shared variable with a float data type and an initial value equal to the test_acc attribute, and assign it to the test_acc attribute + self.test_acc_list=manager.list([]) # create a shared list with an initial value equal to the test_acc_list attribute, using the list method of the manager argument, and assign it to the test_acc_list attribute if self.priority_flag==True: # check if the priority_flag attribute is True self.opt_counter=Array('i',self.opt_counter) # create a shared array with an integer data type and an initial value equal to the opt_counter attribute, and assign it to the opt_counter attribute try: @@ -116,9 +101,9 @@ def init(self,manager): self.nn.bc=manager.list([self.nn.bc]) # try to create a shared list with an initial value equal to a list containing the bc attribute of the nn attribute, using the list method of the manager argument, and assign it to the bc attribute of the nn attribute except Exception: # handle any exception that may occur self.bc_=manager.list() # create an empty shared list using the list method of the manager argument, and assign it to the bc_ attribute - self.epoch_=Value('i',self.epoch_) # create a shared variable with an integer data type and an initial value equal to the epoch_ attribute, and assign it to the epoch_ attribute - self.stop_flag=Value('b',self.stop_flag) # create a shared variable with a boolean data type and an initial value equal to the stop_flag attribute, and assign it to the stop_flag attribute - self.save_flag=Value('b',self.save_flag) # create a shared variable with a boolean data type and an initial value equal to the save_flag attribute, and assign it to the save_flag attribute + self.epoch_=Value('i',0) # create a shared variable with an integer data type and an initial value equal to the epoch_ attribute, and assign it to the epoch_ attribute + self.stop_flag=Value('b',False) # create a shared variable with a boolean data type and an initial value equal to the stop_flag attribute, and assign it to the stop_flag attribute + self.save_flag=Value('b',False) # create a shared variable with a boolean data type and an initial value equal to the save_flag attribute, and assign it to the save_flag attribute self.file_list=manager.list([]) # create an empty shared list using the list method of the manager argument, and assign it to the file_list attribute return # return nothing @@ -459,4 +444,4 @@ def test(self,batch=None): if hasattr(self.nn,'accuracy'): # check if nn attribute has a method named accuracy return test_loss,test_acc # return test_loss and test_acc variables else: # otherwise - return test_loss # return test_loss variable \ No newline at end of file + return test_loss # return test_loss variable From 9ee7d51831e99253623c133a84c165bb2e5c3cea Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 27 Jul 2023 13:12:48 +0800 Subject: [PATCH 093/337] Update kernel.py --- Note 7.0 documentation/RL/kernel/kernel.py | 48 +++++++--------------- 1 file changed, 15 insertions(+), 33 deletions(-) diff --git a/Note 7.0 documentation/RL/kernel/kernel.py b/Note 7.0 documentation/RL/kernel/kernel.py index 664910c8..f8461291 100644 --- a/Note 7.0 documentation/RL/kernel/kernel.py +++ b/Note 7.0 documentation/RL/kernel/kernel.py @@ -15,45 +15,27 @@ def __init__(self,nn=None,process=None): if process!=None: self.reward=np.zeros(process,dtype=np.float32) # the reward array for each process self.sc=np.zeros(process,dtype=np.int32) # the step counter array for each process - self.state_pool={} # the state pool dictionary for each process - self.action_pool={} # the action pool dictionary for each process - self.next_state_pool={} # the next state pool dictionary for each process - self.reward_pool={} # the reward pool dictionary for each process - self.done_pool={} # the done pool dictionary for each process self.epsilon=None # the exploration rate array for each process self.episode_step=None # the maximum number of steps per episode self.pool_size=None # the maximum size of the pool self.batch=None # the batch size for training - self.episode_=0 # the episode counter self.update_step=None # the update frequency for the target network self.trial_count=None # the number of trials to compute the average reward self.process=process # the number of processes - self.process_counter=0 # the running process counter - self.probability_list=[] # the probability list for sampling processes based on their running flag - self.running_flag_list=[] # the running flag list for each process - self.finish_list=[] # the finish flag list for each process - self.running_flag=[] # the running flag array for each process self.PO=None # the parallel optimization mode (1, 2, or 3) self.priority_flag=False # the priority flag for optimization order - self.priority_p=0 # the priority index for optimization order self.max_opt=None # the maximum number of optimization steps per process per episode self.stop=False # the stop flag for training - self.save_flag=False # the save flag for saving parameters to file - self.stop_flag=False # the stop flag for optimization operation - self.opt_counter=None # the optimization counter array for each process self.s=None # the state array for online training mode self.filename='save.dat' # the file name to save parameters to file - self.reward_list=[] # the reward list to store episode rewards - self.loss_list=[] # the loss list to store episode losses - self.total_episode=0 # the total episode counter def init(self,manager): - self.state_pool=manager.dict(self.state_pool) # create a shared memory space for state pool using manager object - self.action_pool=manager.dict(self.action_pool) # create a shared memory space for action pool using manager object - self.next_state_pool=manager.dict(self.next_state_pool) # create a shared memory space for next state pool using manager object - self.reward_pool=manager.dict(self.reward_pool) # create a shared memory space for reward pool using manager object - self.done_pool=manager.dict(self.done_pool) # create a shared memory space for done pool using manager object + self.state_pool=manager.dict({}) # create a shared memory space for state pool using manager object + self.action_pool=manager.dict({}) # create a shared memory space for action pool using manager object + self.next_state_pool=manager.dict({}) # create a shared memory space for next state pool using manager object + self.reward_pool=manager.dict({}) # create a shared memory space for reward pool using manager object + self.done_pool=manager.dict({}) # create a shared memory space for done pool using manager object self.reward=Array('f',self.reward) # create a shared memory space for reward array using Array object if type(self.nn.param[0])!=list: self.loss=np.zeros(self.process,dtype=self.nn.param[0].dtype.name) # create a loss array with the same data type as neural network parameters @@ -61,15 +43,15 @@ def init(self,manager): self.loss=np.zeros(self.process,dtype=self.nn.param[0][0].dtype.name) # create a loss array with the same data type as neural network parameters self.loss=Array('f',self.loss) self.sc=Array('i',self.sc) # create a shared memory space for step counter array using Array object - self.process_counter=Value('i',self.process_counter) # create a shared memory space for process counter using Value object - self.probability_list=manager.list(self.probability_list) # create a shared memory space for probability list using manager object - self.running_flag_list=manager.list(self.running_flag_list) # create a shared memory space for running flag list using manager object - self.finish_list=manager.list(self.finish_list) # create a shared memory space for finish flag list using manager object + self.process_counter=Value('i',0) # create a shared memory space for process counter using Value object + self.probability_list=manager.list([]) # create a shared memory space for probability list using manager object + self.running_flag_list=manager.list([]) # create a shared memory space for running flag list using manager object + self.finish_list=manager.list([]) # create a shared memory space for finish flag list using manager object self.running_flag=manager.list([0]) # create a shared memory space for running flag array using manager object - self.reward_list=manager.list(self.reward_list) # create a shared memory space for reward list using manager object - self.loss_list=manager.list(self.loss_list) # create a shared memory space for loss list using manager object - self.total_episode=Value('i',self.total_episode) # create a shared memory space for total episode counter using Value object - self.priority_p=Value('i',self.priority_p) # create a shared memory space for priority index using Value object + self.reward_list=manager.list([]) # create a shared memory space for reward list using manager object + self.loss_list=manager.list([]) # create a shared memory space for loss list using manager object + self.total_episode=Value('i',0) # create a shared memory space for total episode counter using Value object + self.priority_p=Value('i',0) # create a shared memory space for priority index using Value object if self.priority_flag==True: self.opt_counter=Array('i',np.zeros(self.process,dtype=np.int32)) # create a shared memory space for optimization counter array using Array object try: @@ -85,8 +67,8 @@ def init(self,manager): except Exception: self.bc_=manager.list() # create an empty list to store the exception self.episode_=Value('i',self.total_episode) # create a shared memory space for episode counter using Value object - self.stop_flag=Value('b',self.stop_flag) # create a shared memory space for stop flag using Value object - self.save_flag=Value('b',self.save_flag) # create a shared memory space for save flag using Value object + self.stop_flag=Value('b',False) # create a shared memory space for stop flag using Value object + self.save_flag=Value('b',False) # create a shared memory space for save flag using Value object self.file_list=manager.list([]) # create an empty list to store the file names self.param=manager.dict() # create an empty dictionary to store the parameters return From 855ae2a196d91def9129a39fb5d39b947014b054 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 27 Jul 2023 13:15:15 +0800 Subject: [PATCH 094/337] Update kernel_pytorch.py --- .../RL/kernel/kernel_pytorch.py | 49 ++++++------------- 1 file changed, 16 insertions(+), 33 deletions(-) diff --git a/Note 7.0 documentation/RL/kernel/kernel_pytorch.py b/Note 7.0 documentation/RL/kernel/kernel_pytorch.py index 89248627..80899922 100644 --- a/Note 7.0 documentation/RL/kernel/kernel_pytorch.py +++ b/Note 7.0 documentation/RL/kernel/kernel_pytorch.py @@ -12,59 +12,42 @@ def __init__(self,nn=None,process=None,device='GPU'): self.reward=np.zeros(process,dtype=np.float32) # the reward array for each process self.sc=np.zeros(process,dtype=np.int32) # the step counter array for each process self.device=device # the device to run the model on, either GPU or CPU - self.state_pool={} # the dictionary to store the state transitions for each process - self.action_pool={} # the dictionary to store the actions for each process - self.next_state_pool={} # the dictionary to store the next states for each process - self.reward_pool={} # the dictionary to store the rewards for each process - self.done_pool={} # the dictionary to store the done flags for each process self.epsilon=None # the epsilon value for the epsilon-greedy policy self.episode_step=None # the maximum number of steps per episode self.pool_size=None # the maximum size of the state pool for each process self.batch=None # the batch size for training - self.episode_=0 # the episode counter self.update_step=None # the frequency of updating the target network parameters self.trial_count=None # the number of trials to calculate the average reward self.process=process # the number of processes to run in parallel - self.process_counter=0 # the counter of running processes - self.probability_list=[] # the list of probabilities to select a process - self.running_flag_list=[] # the list of flags to indicate whether a process is running or not - self.finish_list=[] # the list of indices of finished processes - self.running_flag=[] # the list of flags to indicate whether a process is running or not (shared) self.priority_flag=False # the flag to indicate whether to use priority-based optimization or not - self.priority_p=0 # the index of the priority process to optimize first self.max_opt=None # the maximum number of optimization steps for a priority process self.stop=False # the flag to indicate whether to stop training or not - self.save_flag=False # the flag to indicate whether to save the model parameters or not - self.stop_flag=False # the flag to indicate whether to stop training or not (shared) self.opt_counter=None # the array to store the optimization counter for each process self.s=None # a temporary variable to store a state tensor self.filename='save.dat' # the file name to save and load the model parameters and states - self.reward_list=[] # the list of rewards for each episode (shared) - self.loss_list=[] # the list of losses for each episode (shared) - self.total_episode=0 # the total number of episodes (shared) def init(self,manager): """This method is used to initialize some shared variables using a manager object""" - self.state_pool=manager.dict(self.state_pool) - self.action_pool=manager.dict(self.action_pool) - self.next_state_pool=manager.dict(self.next_state_pool) - self.reward_pool=manager.dict(self.reward_pool) - self.done_pool=manager.dict(self.done_pool) + self.state_pool=manager.dict({}) + self.action_pool=manager.dict({}) + self.next_state_pool=manager.dict({}) + self.reward_pool=manager.dict({}) + self.done_pool=manager.dict({}) self.reward=Array('f',self.reward) self.loss=np.zeros(self.process) self.loss=Array('f',self.loss) self.sc=Array('i',self.sc) - self.process_counter=Value('i',self.process_counter) - self.probability_list=manager.list(self.probability_list) - self.running_flag_list=manager.list(self.running_flag_list) - self.finish_list=manager.list(self.finish_list) + self.process_counter=Value('i',0) + self.probability_list=manager.list([]) + self.running_flag_list=manager.list([]) + self.finish_list=manager.list([]) self.running_flag=manager.list([0]) - self.reward_list=manager.list(self.reward_list) - self.loss_list=manager.list(self.loss_list) - self.total_episode=Value('i',self.total_episode) - self.priority_p=Value('i',self.priority_p) + self.reward_list=manager.list([]) + self.loss_list=manager.list([]) + self.total_episode=Value('i',0) + self.priority_p=Value('i',0) if self.priority_flag==True: self.opt_counter=Array('i',np.zeros(self.process,dtype=np.int32)) try: @@ -80,8 +63,8 @@ def init(self,manager): except Exception: self.bc_=manager.list() self.episode_=Value('i',self.total_episode.value) - self.stop_flag=Value('b',self.stop_flag) - self.save_flag=Value('b',self.save_flag) + self.stop_flag=Value('b',False) + self.save_flag=Value('b',False) self.file_list=manager.list([]) return @@ -460,4 +443,4 @@ def train_online(self,p,lock=None,g_lock=None): count=self.nn.counter[p] # get the current value of the counter for this process index count+=1 # increment the counter by 1 self.nn.counter[p]=count # update the counter for this process index - return \ No newline at end of file + return From 0ac9cad605ba4a561c66f24a0de8ec13786f5b65 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 27 Jul 2023 13:28:48 +0800 Subject: [PATCH 095/337] Update and rename kernel.py to kernel.py --- Note 7.0 documentation/DL/kernel/{process => parallel}/kernel.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename Note 7.0 documentation/DL/kernel/{process => parallel}/kernel.py (100%) diff --git a/Note 7.0 documentation/DL/kernel/process/kernel.py b/Note 7.0 documentation/DL/kernel/parallel/kernel.py similarity index 100% rename from Note 7.0 documentation/DL/kernel/process/kernel.py rename to Note 7.0 documentation/DL/kernel/parallel/kernel.py From ca34ebcbe3ef71c1523e460393a38769e2e9053e Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 27 Jul 2023 13:29:06 +0800 Subject: [PATCH 096/337] Update and rename kernel_pytorch.py to kernel_pytorch.py --- .../DL/kernel/{process => parallel}/kernel_pytorch.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename Note 7.0 documentation/DL/kernel/{process => parallel}/kernel_pytorch.py (100%) diff --git a/Note 7.0 documentation/DL/kernel/process/kernel_pytorch.py b/Note 7.0 documentation/DL/kernel/parallel/kernel_pytorch.py similarity index 100% rename from Note 7.0 documentation/DL/kernel/process/kernel_pytorch.py rename to Note 7.0 documentation/DL/kernel/parallel/kernel_pytorch.py From 23e2dc32a76ae888ef8cf6bce8d264820dbc2228 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 27 Jul 2023 13:29:22 +0800 Subject: [PATCH 097/337] Update and rename Note 7.0 documentation/RL/kernel/kernel.py to Note 7.0 documentation/RL/kernel/parallel/kernel.py --- Note 7.0 documentation/RL/kernel/{ => parallel}/kernel.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename Note 7.0 documentation/RL/kernel/{ => parallel}/kernel.py (100%) diff --git a/Note 7.0 documentation/RL/kernel/kernel.py b/Note 7.0 documentation/RL/kernel/parallel/kernel.py similarity index 100% rename from Note 7.0 documentation/RL/kernel/kernel.py rename to Note 7.0 documentation/RL/kernel/parallel/kernel.py From 6f0a380ad0922c017410d04ef28e8b9f3a44ac4e Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 27 Jul 2023 13:29:36 +0800 Subject: [PATCH 098/337] Update and rename Note 7.0 documentation/RL/kernel/kernel_pytorch.py to Note 7.0 documentation/RL/kernel/parallel/kernel_pytorch.py --- Note 7.0 documentation/RL/kernel/{ => parallel}/kernel_pytorch.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename Note 7.0 documentation/RL/kernel/{ => parallel}/kernel_pytorch.py (100%) diff --git a/Note 7.0 documentation/RL/kernel/kernel_pytorch.py b/Note 7.0 documentation/RL/kernel/parallel/kernel_pytorch.py similarity index 100% rename from Note 7.0 documentation/RL/kernel/kernel_pytorch.py rename to Note 7.0 documentation/RL/kernel/parallel/kernel_pytorch.py From c945c98a1f175b7fdcdb365d50c98768dea67947 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 27 Jul 2023 13:31:07 +0800 Subject: [PATCH 099/337] Update kernel_pytorch.py --- Note 7.0 documentation/RL/kernel/parallel/kernel_pytorch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Note 7.0 documentation/RL/kernel/parallel/kernel_pytorch.py b/Note 7.0 documentation/RL/kernel/parallel/kernel_pytorch.py index 80899922..7c2f512f 100644 --- a/Note 7.0 documentation/RL/kernel/parallel/kernel_pytorch.py +++ b/Note 7.0 documentation/RL/kernel/parallel/kernel_pytorch.py @@ -331,11 +331,11 @@ def train(self,p,episode_count,lock,pool_lock): self.running_flag.append(1) # append a 1 to the running flag list to indicate that this process is running self.process_counter.value+=1 # increment the counter of running processes by 1 self.finish_list.append(None) # append a None to the finish list to indicate that this process is not finished yet + lock[1].release() # release the lock for the shared variables try: epsilon=self.epsilon[p] # get the epsilon value for this process index except Exception: epsilon=None # set the epsilon value to None if there is no epsilon value - lock[1].release() # release the lock for the shared variables for k in range(episode_count): # for each episode s=self.nn.env(p=p,initial=True) # get an initial state from the environment according to this process index s=np.array(s) # convert the state to an array From 9cf8045b730a03788c2a279d5bff7b8db2daddae Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 27 Jul 2023 13:31:24 +0800 Subject: [PATCH 100/337] Update kernel.py --- Note 7.0 documentation/RL/kernel/parallel/kernel.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Note 7.0 documentation/RL/kernel/parallel/kernel.py b/Note 7.0 documentation/RL/kernel/parallel/kernel.py index f8461291..4d07e1c3 100644 --- a/Note 7.0 documentation/RL/kernel/parallel/kernel.py +++ b/Note 7.0 documentation/RL/kernel/parallel/kernel.py @@ -482,14 +482,14 @@ def train(self,p,episode_count,lock,pool_lock,g_lock=None): self.running_flag.append(1) # append a running flag of 1 to indicate that current process is running self.process_counter.value+=1 # increment the process counter by 1 self.finish_list.append(None) # append a finish flag of None to indicate that current process is not finished - try: - epsilon=self.epsilon[p] # get the exploration rate for current process - except Exception: - epsilon=None # set the exploration rate as None if exception occurs if self.PO==1 or self.PO==2: # check if parallel optimization mode is 1 or 2 lock[1].release() # release the lock for initialization operation elif self.PO==3: # check if parallel optimization mode is 3 lock[1].release() # release the lock for initialization operation + try: + epsilon=self.epsilon[p] # get the exploration rate for current process + except Exception: + epsilon=None # set the exploration rate as None if exception occurs for k in range(episode_count): # loop over each episode s=self.nn.env(p=p,initial=True) # reset the environment and get the initial state if type(self.nn.param[0])!=list: From 6efe9b053f659ca9f0e7fa487aab8840744e14ab Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 27 Jul 2023 13:39:32 +0800 Subject: [PATCH 101/337] Add files via upload --- .../tensorflow/non-parallel/bert.py | 167 ++++++++++++++++++ .../tensorflow/non-parallel/cnn.py | 27 +++ .../tensorflow/non-parallel/lstm.py | 26 +++ .../tensorflow/non-parallel/nn.py | 23 +++ .../tensorflow/non-parallel/nn_acc.py | 28 +++ .../tensorflow/non-parallel/nn_ol.py | 37 ++++ .../tensorflow/parallel/LSTM.py | 46 +++++ .../tensorflow/parallel/attn_LSTM.py | 53 ++++++ .../neural network/tensorflow/parallel/cnn.py | 59 +++++++ .../tensorflow/parallel/cnn_lmix.py | 67 +++++++ .../neural network/tensorflow/parallel/nn.py | 39 ++++ .../neural network/tensorflow/parallel/nn_.py | 39 ++++ .../tensorflow/parallel/nn_acc.py | 44 +++++ .../tensorflow/parallel/nn_attenuate.py | 49 +++++ .../tensorflow/parallel/nn_clipping.py | 47 +++++ .../tensorflow/parallel/nn_device.py | 40 +++++ .../tensorflow/parallel/self_LSTM.py | 54 ++++++ .../tensorflow/parallel/transformer.py | 55 ++++++ 18 files changed, 900 insertions(+) create mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/non-parallel/bert.py create mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/non-parallel/cnn.py create mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/non-parallel/lstm.py create mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/non-parallel/nn.py create mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/non-parallel/nn_acc.py create mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/non-parallel/nn_ol.py create mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/parallel/LSTM.py create mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/parallel/attn_LSTM.py create mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/parallel/cnn.py create mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/parallel/cnn_lmix.py create mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn.py create mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn_.py create mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn_acc.py create mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn_attenuate.py create mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn_clipping.py create mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn_device.py create mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/parallel/self_LSTM.py create mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/parallel/transformer.py diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/non-parallel/bert.py b/Note 7.0 documentation/DL/neural network/tensorflow/non-parallel/bert.py new file mode 100644 index 00000000..9715477f --- /dev/null +++ b/Note 7.0 documentation/DL/neural network/tensorflow/non-parallel/bert.py @@ -0,0 +1,167 @@ +import tensorflow as tf +import tensorflow_hub as hub + + +class bert: + def __init__(self): + text_input=tf.keras.layers.Input(shape=(),dtype=tf.string,name='text') + preprocessing_layer=hub.KerasLayer(tfhub_handle_preprocess,name='preprocessing') + encoder_inputs=preprocessing_layer(text_input) + encoder=hub.KerasLayer(tfhub_handle_encoder,trainable=True,name='BERT_encoder') + outputs=encoder(encoder_inputs) + net=outputs['pooled_output'] + net=tf.keras.layers.Dropout(0.1)(net) + net=tf.keras.layers.Dense(1,activation=None,name='classifier')(net) + self.model=tf.keras.Model(text_input,net) + self.param=self.model.weights + self.opt=tf.keras.optimizers.Adam() + + + def fp(self,data): + return self.model(data) + + + def loss(self,output,labels): + bce=tf.keras.losses.BinaryCrossentropy(from_logits=True) + return bce(output,labels) + + +bert_model_name = 'small_bert/bert_en_uncased_L-4_H-512_A-8' +map_name_to_handle = { + 'bert_en_uncased_L-12_H-768_A-12': + 'https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/3', + 'bert_en_cased_L-12_H-768_A-12': + 'https://tfhub.dev/tensorflow/bert_en_cased_L-12_H-768_A-12/3', + 'bert_multi_cased_L-12_H-768_A-12': + 'https://tfhub.dev/tensorflow/bert_multi_cased_L-12_H-768_A-12/3', + 'small_bert/bert_en_uncased_L-2_H-128_A-2': + 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-2_H-128_A-2/1', + 'small_bert/bert_en_uncased_L-2_H-256_A-4': + 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-2_H-256_A-4/1', + 'small_bert/bert_en_uncased_L-2_H-512_A-8': + 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-2_H-512_A-8/1', + 'small_bert/bert_en_uncased_L-2_H-768_A-12': + 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-2_H-768_A-12/1', + 'small_bert/bert_en_uncased_L-4_H-128_A-2': + 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-128_A-2/1', + 'small_bert/bert_en_uncased_L-4_H-256_A-4': + 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-256_A-4/1', + 'small_bert/bert_en_uncased_L-4_H-512_A-8': + 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-512_A-8/1', + 'small_bert/bert_en_uncased_L-4_H-768_A-12': + 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-768_A-12/1', + 'small_bert/bert_en_uncased_L-6_H-128_A-2': + 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-6_H-128_A-2/1', + 'small_bert/bert_en_uncased_L-6_H-256_A-4': + 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-6_H-256_A-4/1', + 'small_bert/bert_en_uncased_L-6_H-512_A-8': + 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-6_H-512_A-8/1', + 'small_bert/bert_en_uncased_L-6_H-768_A-12': + 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-6_H-768_A-12/1', + 'small_bert/bert_en_uncased_L-8_H-128_A-2': + 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-8_H-128_A-2/1', + 'small_bert/bert_en_uncased_L-8_H-256_A-4': + 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-8_H-256_A-4/1', + 'small_bert/bert_en_uncased_L-8_H-512_A-8': + 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-8_H-512_A-8/1', + 'small_bert/bert_en_uncased_L-8_H-768_A-12': + 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-8_H-768_A-12/1', + 'small_bert/bert_en_uncased_L-10_H-128_A-2': + 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-10_H-128_A-2/1', + 'small_bert/bert_en_uncased_L-10_H-256_A-4': + 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-10_H-256_A-4/1', + 'small_bert/bert_en_uncased_L-10_H-512_A-8': + 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-10_H-512_A-8/1', + 'small_bert/bert_en_uncased_L-10_H-768_A-12': + 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-10_H-768_A-12/1', + 'small_bert/bert_en_uncased_L-12_H-128_A-2': + 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-12_H-128_A-2/1', + 'small_bert/bert_en_uncased_L-12_H-256_A-4': + 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-12_H-256_A-4/1', + 'small_bert/bert_en_uncased_L-12_H-512_A-8': + 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-12_H-512_A-8/1', + 'small_bert/bert_en_uncased_L-12_H-768_A-12': + 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-12_H-768_A-12/1', + 'albert_en_base': + 'https://tfhub.dev/tensorflow/albert_en_base/2', + 'electra_small': + 'https://tfhub.dev/google/electra_small/2', + 'electra_base': + 'https://tfhub.dev/google/electra_base/2', + 'experts_pubmed': + 'https://tfhub.dev/google/experts/bert/pubmed/2', + 'experts_wiki_books': + 'https://tfhub.dev/google/experts/bert/wiki_books/2', + 'talking-heads_base': + 'https://tfhub.dev/tensorflow/talkheads_ggelu_bert_en_base/1', +} +map_model_to_preprocess = { + 'bert_en_uncased_L-12_H-768_A-12': + 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', + 'bert_en_cased_L-12_H-768_A-12': + 'https://tfhub.dev/tensorflow/bert_en_cased_preprocess/3', + 'small_bert/bert_en_uncased_L-2_H-128_A-2': + 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', + 'small_bert/bert_en_uncased_L-2_H-256_A-4': + 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', + 'small_bert/bert_en_uncased_L-2_H-512_A-8': + 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', + 'small_bert/bert_en_uncased_L-2_H-768_A-12': + 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', + 'small_bert/bert_en_uncased_L-4_H-128_A-2': + 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', + 'small_bert/bert_en_uncased_L-4_H-256_A-4': + 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', + 'small_bert/bert_en_uncased_L-4_H-512_A-8': + 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', + 'small_bert/bert_en_uncased_L-4_H-768_A-12': + 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', + 'small_bert/bert_en_uncased_L-6_H-128_A-2': + 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', + 'small_bert/bert_en_uncased_L-6_H-256_A-4': + 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', + 'small_bert/bert_en_uncased_L-6_H-512_A-8': + 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', + 'small_bert/bert_en_uncased_L-6_H-768_A-12': + 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', + 'small_bert/bert_en_uncased_L-8_H-128_A-2': + 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', + 'small_bert/bert_en_uncased_L-8_H-256_A-4': + 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', + 'small_bert/bert_en_uncased_L-8_H-512_A-8': + 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', + 'small_bert/bert_en_uncased_L-8_H-768_A-12': + 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', + 'small_bert/bert_en_uncased_L-10_H-128_A-2': + 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', + 'small_bert/bert_en_uncased_L-10_H-256_A-4': + 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', + 'small_bert/bert_en_uncased_L-10_H-512_A-8': + 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', + 'small_bert/bert_en_uncased_L-10_H-768_A-12': + 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', + 'small_bert/bert_en_uncased_L-12_H-128_A-2': + 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', + 'small_bert/bert_en_uncased_L-12_H-256_A-4': + 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', + 'small_bert/bert_en_uncased_L-12_H-512_A-8': + 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', + 'small_bert/bert_en_uncased_L-12_H-768_A-12': + 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', + 'bert_multi_cased_L-12_H-768_A-12': + 'https://tfhub.dev/tensorflow/bert_multi_cased_preprocess/3', + 'albert_en_base': + 'https://tfhub.dev/tensorflow/albert_en_preprocess/3', + 'electra_small': + 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', + 'electra_base': + 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', + 'experts_pubmed': + 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', + 'experts_wiki_books': + 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', + 'talking-heads_base': + 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', +} +tfhub_handle_encoder=map_name_to_handle[bert_model_name] +tfhub_handle_preprocess=map_model_to_preprocess[bert_model_name] \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/non-parallel/cnn.py b/Note 7.0 documentation/DL/neural network/tensorflow/non-parallel/cnn.py new file mode 100644 index 00000000..bd7ad1bf --- /dev/null +++ b/Note 7.0 documentation/DL/neural network/tensorflow/non-parallel/cnn.py @@ -0,0 +1,27 @@ +import tensorflow as tf +from tensorflow.keras import layers,models + + +class cnn: + def __init__(self): + self.model=models.Sequential() + self.model.add(layers.Conv2D(32,(3,3),activation='relu',input_shape=(32,32,3))) + self.model.add(layers.MaxPooling2D((2,2))) + self.model.add(layers.Conv2D(64,(3,3),activation='relu')) + self.model.add(layers.MaxPooling2D((2,2))) + self.model.add(layers.Conv2D(64,(3,3),activation='relu')) + self.model.add(layers.Flatten()) + self.model.add(layers.Dense(64,activation='relu')) + self.model.add(layers.Dense(10)) + self.param=self.model.weights + self.opt=tf.keras.optimizers.Adam() + + + def fp(self,data): + output=self.model(data) + return output + + + def loss(self,output,labels): + loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + return loss(labels,output) \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/non-parallel/lstm.py b/Note 7.0 documentation/DL/neural network/tensorflow/non-parallel/lstm.py new file mode 100644 index 00000000..29b9b396 --- /dev/null +++ b/Note 7.0 documentation/DL/neural network/tensorflow/non-parallel/lstm.py @@ -0,0 +1,26 @@ +import tensorflow as tf + + +class lstm: + def __init__(self,encoder): + self.model=tf.keras.Sequential([ + encoder, + tf.keras.layers.Embedding(len(encoder.get_vocabulary()), 64, mask_zero=True), + tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(64, return_sequences=True)), + tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(32)), + tf.keras.layers.Dense(64, activation='relu'), + tf.keras.layers.Dropout(0.5), + tf.keras.layers.Dense(1) + ]) + self.param=self.model.weights[1:] + self.opt=tf.keras.optimizers.Adam() + + + def fp(self,data): + output=self.model(data) + return output + + + def loss(self,output,labels): + bce=tf.keras.losses.BinaryCrossentropy(from_logits=True) + return bce(labels,output) \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/non-parallel/nn.py b/Note 7.0 documentation/DL/neural network/tensorflow/non-parallel/nn.py new file mode 100644 index 00000000..b4f03a92 --- /dev/null +++ b/Note 7.0 documentation/DL/neural network/tensorflow/non-parallel/nn.py @@ -0,0 +1,23 @@ +import tensorflow as tf + + +class nn: + def __init__(self): + self.model=tf.keras.models.Sequential([ + tf.keras.layers.Flatten(input_shape=(28, 28)), + tf.keras.layers.Dense(128,activation='relu'), + tf.keras.layers.Dropout(0.2), + tf.keras.layers.Dense(10) + ]) + self.param=self.model.weights #parameter list,kernel uses it list for backpropagation. + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.opt=tf.keras.optimizers.Adam() #optimizer,kernel uses it to optimize. + + + def fp(self,data): #forward propagation function,kernel uses it for forward propagation. + output=self.model(data) + return output + + + def loss(self,output,labels): #loss functino,kernel uses it to calculate loss. + return self.loss_object(labels,output) \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/non-parallel/nn_acc.py b/Note 7.0 documentation/DL/neural network/tensorflow/non-parallel/nn_acc.py new file mode 100644 index 00000000..0a726603 --- /dev/null +++ b/Note 7.0 documentation/DL/neural network/tensorflow/non-parallel/nn_acc.py @@ -0,0 +1,28 @@ +import tensorflow as tf + +#An example with accuracy function. +class nn: + def __init__(self): + self.model=tf.keras.models.Sequential([ + tf.keras.layers.Flatten(input_shape=(28, 28)), + tf.keras.layers.Dense(128,activation='relu'), + tf.keras.layers.Dropout(0.2), + tf.keras.layers.Dense(10) + ]) + self.param=self.model.weights #parameter list,kernel uses it list for backpropagation. + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.train_accuracy=tf.keras.metrics.SparseCategoricalAccuracy(name='train_accuracy') + self.opt=tf.keras.optimizers.Adam() #optimizer,kernel uses it to optimize. + + + def fp(self,data): #forward propagation function,kernel uses it for forward propagation. + output=self.model(data) + return output + + + def loss(self,output,labels): #loss functino,kernel uses it to calculate loss. + return self.loss_object(labels,output) + + + def accuracy(self,output,labels): #accuracy function,kernel uses it to calculate accuracy. + return self.train_accuracy(labels,output) \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/non-parallel/nn_ol.py b/Note 7.0 documentation/DL/neural network/tensorflow/non-parallel/nn_ol.py new file mode 100644 index 00000000..9dfa608d --- /dev/null +++ b/Note 7.0 documentation/DL/neural network/tensorflow/non-parallel/nn_ol.py @@ -0,0 +1,37 @@ +import tensorflow as tf # import TensorFlow library +import numpy as np # import NumPy library + +# This is an example of online training. +class nn: # define a class for the neural network + def __init__(self,data,labels): # initialize the network with data and labels + self.train_data=data # store the training data + self.train_labels=labels # store the training labels + self.model=tf.keras.models.Sequential([ # create a sequential model with four layers + tf.keras.layers.Flatten(input_shape=(28, 28)), # flatten the input to a vector of 28*28 elements + tf.keras.layers.Dense(128,activation='relu'), # add a dense layer with 128 units and ReLU activation + tf.keras.layers.Dropout(0.2), # add a dropout layer with 0.2 dropout rate to prevent overfitting + tf.keras.layers.Dense(10) # add a dense layer with 10 units for the output logits + ]) + self.param=self.model.weights # parameter list, kernel uses it list for backpropagation + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) # loss object, kernel uses it to calculate loss. Here we use sparse categorical crossentropy with logits as output + self.opt=tf.keras.optimizers.Adam() # optimizer, kernel uses it to optimize. Here we use Adam optimizer + self.train_loss_list=[] # a list to store the training loss values + self.counter=0 # counter to keep track of the number of online updates + self.max_length=1000 # maximum length of train_loss_list + + + def online(self): # This is simulative online function, kernel uses it to get online data and labels + index=np.random.choice(60000,size=[32]) # randomly sample 32 indices from 0 to 59999 + if self.counter==10000: # if the counter reaches 10000, stop the online training + return 'stop' + else: # otherwise, return the data and labels corresponding to the sampled indices + return [self.train_data[index],self.train_labels[index]] + + + def fp(self,data): # forward propagation function, kernel uses it for forward propagation + output=self.model(data) # pass the data through the model and get the output logits + return output + + + def loss(self,output,labels): # loss function, kernel uses it to calculate loss + return self.loss_object(labels,output) # return the sparse categorical crossentropy loss between labels and output \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/parallel/LSTM.py b/Note 7.0 documentation/DL/neural network/tensorflow/parallel/LSTM.py new file mode 100644 index 00000000..e8686c10 --- /dev/null +++ b/Note 7.0 documentation/DL/neural network/tensorflow/parallel/LSTM.py @@ -0,0 +1,46 @@ +import tensorflow as tf +from Note.nn.layer.LSTM import LSTM +from Note.nn.layer.dense import dense +from Note.nn.parallel.optimizer import Adam + +# Define a recurrent neural network class with LSTM layers +class lstm: + def __init__(self): + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.optimizer=Adam() + # Initialize a variable to keep track of the batch count + self.bc=tf.Variable(0, dtype=tf.float32) + + + def build(self): + # Create two LSTM layers with 50 hidden units each + self.lstm1=LSTM(weight_shape=(3,50),return_sequence=True) # First LSTM layer that returns the full sequence + self.lstm2=LSTM(weight_shape=(50,50),) # Second LSTM layer that returns the last output only + # Create a dense layer with 10 output units + self.dense=dense([50, 10]) + # Store the parameters of the layers in a list + self.param=[self.lstm1.param, + self.lstm2.param, + self.dense.param, + ] + return + + + def fp(self,data): + # Perform forward propagation on the input data + x=self.lstm1.output(data) # First LSTM layer output + x=self.lstm2.output(x) # Second LSTM layer output + output=self.dense.output(x) # Dense layer output + return output + + + def loss(self, output, labels): + # Compute the loss between the output and the labels + return self.loss_object(labels, output) + + + def opt(self,gradient): + # Perform optimization on the parameters using the gradient and the batch count + param=self.optimizer.opt(gradient,self.param,self.bc[0]) + return param \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/parallel/attn_LSTM.py b/Note 7.0 documentation/DL/neural network/tensorflow/parallel/attn_LSTM.py new file mode 100644 index 00000000..198e46b1 --- /dev/null +++ b/Note 7.0 documentation/DL/neural network/tensorflow/parallel/attn_LSTM.py @@ -0,0 +1,53 @@ +import tensorflow as tf +from Note.nn.layer.LSTM import LSTM +from Note.nn.layer.dense import dense +from Note.nn.layer.attention import attention +from Note.nn.parallel.optimizer import Adam + +# Define a recurrent neural network class with LSTM layers and attention +class attn_lstm: + def __init__(self): + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.optimizer=Adam() + # Initialize a variable to keep track of the batch count + self.bc=tf.Variable(0, dtype=tf.float32) + + + def build(self): + # Create two LSTM layers with 50 hidden units each + self.lstm1=LSTM(weight_shape=(3,50),return_sequence=True) # First LSTM layer that returns the full sequence + self.lstm2=LSTM(weight_shape=(50,50),) # Second LSTM layer that returns the last output only + # Create a dense layer with 10 output units and softmax activation + self.dense=dense([100, 10],activation='softmax') + # Create an attention object with weight shape (50, 50) + self.attn=attention((50,50)) + # Store the parameters of the layers and the attention in a list + self.param=[self.lstm1.param, + self.lstm2.param, + self.dense.param, + self.attn.param] # Add the attention parameters to the list + return + + + def fp(self,data): + # Perform forward propagation on the input data + x=self.lstm1.output(data) # First LSTM layer output, shape (B, N, 50) + y=self.lstm2.output(x) # Second LSTM layer output, shape (B, 50) + # Use the attention object to compute the context vector from x and y, shape (B, 50) + context_vector,_,_=self.attn.output(x,y) + # Concatenate the context vector and the second LSTM output, shape (B, 100) + z=tf.concat([context_vector,y],axis=-1) + output=self.dense.output(z) # Dense layer output with softmax activation, shape (B, 10) + return output + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) + + + def opt(self,gradient): + # Perform optimization on the parameters using the gradient and the batch count + param=self.optimizer.opt(gradient,self.param,self.bc[0]) + return param \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/parallel/cnn.py b/Note 7.0 documentation/DL/neural network/tensorflow/parallel/cnn.py new file mode 100644 index 00000000..59be2e63 --- /dev/null +++ b/Note 7.0 documentation/DL/neural network/tensorflow/parallel/cnn.py @@ -0,0 +1,59 @@ +import tensorflow as tf +from Note.nn.layer.conv2d import conv2d +from Note.nn.layer.dense import dense +from Note.nn.layer.flatten import flatten +from Note.nn.parallel.optimizer import Adam + +# Define a convolutional neural network class +class cnn: + def __init__(self): + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.optimizer=Adam() + # Initialize a variable to keep track of the batch count + self.bc=tf.Variable(0,dtype=tf.float32) + + + def build(self): + # Create three convolutional layers with relu activations + self.conv1=conv2d(weight_shape=(3,3,3,32),activation='relu') + self.conv2=conv2d(weight_shape=(3,3,32,64),activation='relu') + self.conv3=conv2d(weight_shape=(3,3,64,64),activation='relu') + # Create two dense layers with relu and linear activations + self.dense1=dense([64*4*4,64],activation='relu') + self.dense2=dense([64,10]) + # Store the parameters of the layers in a list + self.param = [self.conv1.weight, + self.conv2.weight, + self.conv3.weight, + self.dense1.weight, + self.dense1.bias, + self.dense2.weight, + self.dense2.bias] + return + + + def fp(self,data): + # Perform forward propagation on the input data + x=tf.nn.conv2d(data,self.conv1.weight,strides=(1,1),padding='SAME') # First convolutional layer + x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # First max pooling layer + x=tf.nn.conv2d(x,self.conv2.weight,strides=(1,1),padding='SAME') # Second convolutional layer + x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # Second max pooling layer + x=tf.nn.conv2d(x,self.conv3.weight,strides=(1,1),padding='SAME') # Third convolutional layer + x=flatten(x) # Flatten the output to a vector + x=tf.nn.relu(tf.matmul(x,self.dense1.weight)+self.dense1.bias) # First dense layer with relu activation + x=tf.nn.dropout(x,rate=0.5) # Apply dropout to prevent overfitting + output=tf.matmul(x,self.dense2.weight)+self.dense2.bias # Output layer with linear activation + output=tf.nn.dropout(output,rate=0.5) # Apply dropout to prevent overfitting + return output + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) + + + def opt(self,gradient): + # Perform optimization on the parameters using the gradient and the batch count + param=self.optimizer.opt(gradient,self.param,self.bc[0]) + return param \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/parallel/cnn_lmix.py b/Note 7.0 documentation/DL/neural network/tensorflow/parallel/cnn_lmix.py new file mode 100644 index 00000000..b6e66458 --- /dev/null +++ b/Note 7.0 documentation/DL/neural network/tensorflow/parallel/cnn_lmix.py @@ -0,0 +1,67 @@ +import tensorflow as tf +from Note.nn.layer.conv2d import conv2d +from Note.nn.layer.dense import dense +from Note.nn.layer.flatten import flatten +from Note.nn.parallel.optimizer import Adam +from Note.nn.layer.LMix import lmix + +# Define a convolutional neural network class with mixup augmentation +class cnn: + def __init__(self): + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.optimizer=Adam() + # Initialize the alpha parameter for mixup + self.alpha=1.0 + # Initialize a variable to keep track of the batch count + self.bc=tf.Variable(0,dtype=tf.float32) + + + def build(self): + # Create three convolutional layers with relu activations + self.conv1=conv2d(weight_shape=(3,3,3,32),activation='relu') + self.conv2=conv2d(weight_shape=(3,3,32,64),activation='relu') + self.conv3=conv2d(weight_shape=(3,3,64,64),activation='relu') + # Create two dense layers with relu and linear activations + self.dense1=dense([64*4*4,64],activation='relu') + self.dense2=dense([64,10]) + # Store the parameters of the layers in a list + self.param = [self.conv1.weight, + self.conv2.weight, + self.conv3.weight, + self.dense1.weight, + self.dense1.bias, + self.dense2.weight, + self.dense2.bias] + return + + + def data_func(self,data_batch,labels_batch): + # Apply mixup augmentation on the input data and labels + return lmix(data_batch,labels_batch,self.alpha) + + + def fp(self,data): + # Perform forward propagation on the input data + x=tf.nn.conv2d(data,self.conv1.weight,strides=(1,1),padding='SAME') # First convolutional layer + x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # First max pooling layer + x=tf.nn.conv2d(x,self.conv2.weight,strides=(1,1),padding='SAME') # Second convolutional layer + x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # Second max pooling layer + x=tf.nn.conv2d(x,self.conv3.weight,strides=(1,1),padding='SAME') # Third convolutional layer + x=flatten(x) # Flatten the output to a vector + x=tf.nn.relu(tf.matmul(x,self.dense1.weight)+self.dense1.bias) # First dense layer with relu activation + x=tf.nn.dropout(x,rate=0.5) # Apply dropout to prevent overfitting + output=tf.matmul(x,self.dense2.weight)+self.dense2.bias # Output layer with linear activation + output=tf.nn.dropout(output,rate=0.5) # Apply dropout to prevent overfitting + return output + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) + + + def opt(self,gradient): + # Perform optimization on the parameters using the gradient and the batch count + param=self.optimizer.opt(gradient,self.param,self.bc[0]) + return param \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn.py b/Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn.py new file mode 100644 index 00000000..1b16b5c0 --- /dev/null +++ b/Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn.py @@ -0,0 +1,39 @@ +import tensorflow as tf +import Note.nn.layer.dense as d +from Note.nn.parallel.optimizer import Momentum +from Note.nn.layer.flatten import flatten + +# Define a neural network class +class nn: + def __init__(self): + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.optimizer=Momentum(0.07,0.7) + + + def build(self): + # Create two dense layers with relu and linear activations + self.layer1=d.dense([784,128],activation='relu') + self.layer2=d.dense([128,10]) + # Store the parameters of the layers in a list + self.param=[self.layer1.param,self.layer2.param] + return + + + def fp(self,data): + # Perform forward propagation on the input data + data=flatten(data) + output1=self.layer1.output(data) + output2=self.layer2.output(output1) + return output2 + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) + + + def opt(self,gradient): + # Perform optimization on the parameters using the gradient + param=self.optimizer.opt(gradient,self.param) + return param \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn_.py b/Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn_.py new file mode 100644 index 00000000..d115cc0c --- /dev/null +++ b/Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn_.py @@ -0,0 +1,39 @@ +import tensorflow as tf +from Note.nn.parallel.optimizer import Momentum +from Note.nn.layer.flatten import flatten + +# Define a neural network class +class nn: + def __init__(self): + # Initialize the weights and biases of three layers with random values + self.weight1=tf.Variable(tf.random.normal([784,64],dtype='float64')) + self.bias1=tf.Variable(tf.random.normal([64],dtype='float64')) + self.weight2=tf.Variable(tf.random.normal([64,64],dtype='float64')) + self.bias2=tf.Variable(tf.random.normal([64],dtype='float64')) + self.weight3=tf.Variable(tf.random.normal([64,10],dtype='float64')) + self.bias3=tf.Variable(tf.random.normal([10],dtype='float64')) + # Store the parameters of the layers in a list + self.param=[self.weight1,self.weight2,self.weight3,self.bias1,self.bias2,self.bias3] + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.optimizer=Momentum(0.07,0.7) + + + def fp(self,data): + # Perform forward propagation on the input data + data=flatten(data) + layer1=tf.nn.relu(tf.matmul(data,self.weight1)+self.bias1) # First layer with relu activation + layer2=tf.nn.relu(tf.matmul(layer1,self.weight2)+self.bias2) # Second layer with relu activation + output=tf.matmul(layer2,self.weight3)+self.bias3 # Output layer with linear activation + return output + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) + + + def opt(self,gradient): + # Perform optimization on the parameters using the gradient + param=self.optimizer.opt(gradient,self.param) + return param \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn_acc.py b/Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn_acc.py new file mode 100644 index 00000000..19b055dc --- /dev/null +++ b/Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn_acc.py @@ -0,0 +1,44 @@ +import tensorflow as tf +import Note.nn.layer.dense as d +from Note.nn.parallel.optimizer import Momentum +from Note.nn.layer.flatten import flatten +from Note.nn.accuracy import sparse_categorical_accuracy + +# Define a neural network class +class nn: + def __init__(self): + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.optimizer=Momentum(0.07,0.7) + + + def build(self): + # Create two dense layers with relu and linear activations + self.layer1=d.dense([784,128],activation='relu') + self.layer2=d.dense([128,10]) + # Store the parameters of the layers in a list + self.param=[self.layer1.param,self.layer2.param] + return + + + def fp(self,data): + # Perform forward propagation on the input data + data=flatten(data) + output1=self.layer1.output(data) + output2=self.layer2.output(output1) + return output2 + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) + + + def accuracy(self,output,labels): #accuracy function,kernel uses it to calculate accuracy. + return sparse_categorical_accuracy(labels,output) + + + def opt(self,gradient): + # Perform optimization on the parameters using the gradient + param=self.optimizer.opt(gradient,self.param) + return param \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn_attenuate.py b/Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn_attenuate.py new file mode 100644 index 00000000..24d2836b --- /dev/null +++ b/Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn_attenuate.py @@ -0,0 +1,49 @@ +import tensorflow as tf +import Note.nn.layer.dense as d +from Note.nn.parallel.optimizer import Momentum +from Note.nn.layer.flatten import flatten + +# Define a neural network class with gradient attenuation +class nn: + def __init__(self): + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.optimizer=Momentum(0.07,0.7) + # Initialize a variable to keep track of the number of optimization steps + self.opt_counter=tf.Variable(tf.zeros(7,dtype=tf.float32)) + + + def build(self): + # Create two dense layers with relu and linear activations + self.layer1=d.dense([784,128],activation='relu') + self.layer2=d.dense([128,10]) + # Store the parameters of the layers in a list + self.param=[self.layer1.param,self.layer2.param] + return + + + def fp(self,data): + # Perform forward propagation on the input data + data=flatten(data) + output1=self.layer1.output(data) + output2=self.layer2.output(output1) + return output2 + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) + + + def attenuate(self,gradient,oc,p): #gradient attenuation function,kernel uses it to calculate attenuation coefficient. + # Apply an exponential decay to the gradient based on the optimization counter + ac=0.9**oc[p] #ac:attenuation coefficient + for i in range(len(gradient)): #oc:optimization counter + gradient[i]=ac*gradient[i] #p:process number + return gradient + + + def opt(self,gradient): + # Perform optimization on the parameters using the gradient + param=self.optimizer.opt(gradient,self.param) + return param \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn_clipping.py b/Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn_clipping.py new file mode 100644 index 00000000..a0f757d3 --- /dev/null +++ b/Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn_clipping.py @@ -0,0 +1,47 @@ +import tensorflow as tf +import Note.nn.layer.dense as d +from Note.nn.parallel.optimizer import Momentum +from Note.nn.layer.flatten import flatten +from Note.nn.gradient_clipping import gradient_clipping + +# Define a neural network class +class nn: + def __init__(self): + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.optimizer=Momentum(0.07,0.7) + + + def build(self): + # Create two dense layers with relu and linear activations + self.layer1=d.dense([784,128],activation='relu') + self.layer2=d.dense([128,10]) + # Store the parameters of the layers in a list + self.param=[self.layer1.param,self.layer2.param] + return + + + def fp(self,data): + # Perform forward propagation on the input data + data=flatten(data) + output1=self.layer1.output(data) + output2=self.layer2.output(output1) + return output2 + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) + + + def gradient(self,tape,loss): + # Compute the gradients of the loss with respect to the parameters + grads=tape.gradient(loss,self.param) + # Clip the gradients by value to avoid exploding gradients + return gradient_clipping(grads,'value',1.0) + + + def opt(self,gradient): + # Perform optimization on the parameters using the gradient + param=self.optimizer.opt(gradient,self.param) + return param \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn_device.py b/Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn_device.py new file mode 100644 index 00000000..16ef49eb --- /dev/null +++ b/Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn_device.py @@ -0,0 +1,40 @@ +import tensorflow as tf # import TensorFlow library +import Note.nn.layer.dense as d # import Note's dense layer module +from Note.nn.layer.flatten import flatten # import Note's flatten layer function +from Note.nn.parallel.optimizer import Momentum # import Note's momentum optimizer module +from Note.nn.parallel.assign_device import assign_device # import the function to assign device according to the process index and the device type + + +class nn: # A neural network class example, allocate device for multiple threads + def __init__(self): # initialize the network + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) # loss object, kernel uses it to calculate loss. Here we use sparse categorical crossentropy with logits as output + self.optimizer=Momentum(0.07,0.7) # optimizer, kernel uses it to optimize. Here we use a custom momentum optimizer with learning rate 0.07 and momentum 0.7 + self.info='example' # some information about the network + + + def build(self): # build function, kernel uses it to create the network layers + # Create two dense layers with relu and linear activations + self.layer1=d.dense([784,128],activation='relu') # the first layer with 784 input units and 128 output units and ReLU activation + self.layer2=d.dense([128,10]) # the second layer with 128 input units and 10 output units and linear activation + # Store the parameters of the layers in a list + self.param=[self.layer1.param,self.layer2.param] # parameter list of both layers, kernel uses it list for backpropagation + return + + + def fp(self,data,p): # forward propagation function, kernel uses it for forward propagation + with tf.device(assign_device(p,'GPU')): # assign the device according to the process index p + data=flatten(data) # flatten the data to a vector of 784 elements + output1=self.layer1.output(data) # pass the data through the first layer and get the output + output2=self.layer2.output(output1) # pass the output of the first layer through the second layer and get the final output logits + return output2 + + + def loss(self,output,labels,p): # loss function, kernel uses it to calculate loss + with tf.device(assign_device(p,'GPU')): # assign the device according to the process index p + return self.loss_object(labels,output) # return the mean softmax cross entropy loss between labels and output + + + def opt(self,gradient): # optimization function, kernel uses it to optimize parameter + # Perform optimization on the parameters using the gradient + param=self.optimizer.opt(gradient,self.param) # apply the Note's momentum optimizer to update the parameters using the gradient + return param \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/parallel/self_LSTM.py b/Note 7.0 documentation/DL/neural network/tensorflow/parallel/self_LSTM.py new file mode 100644 index 00000000..85af312a --- /dev/null +++ b/Note 7.0 documentation/DL/neural network/tensorflow/parallel/self_LSTM.py @@ -0,0 +1,54 @@ +import tensorflow as tf +from Note.nn.parallel.optimizer import Adam +from Note.nn.layer.LSTM import LSTM +from Note.nn.layer.self_attention import self_attention +from Note.nn.layer.dense import dense +from Note.nn.layer.emdedding import embedding + +# Define a LSTM self attention neural network class +class self_LSTM: + def __init__(self,vocab_size,embed_size,num_heads,num_layers): + self.vocab_size=vocab_size + self.embed_size=embed_size + self.num_heads=num_heads + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.optimizer=Adam() + # Initialize a variable to keep track of the batch count + self.bc=tf.Variable(0,dtype=tf.float32) + + + def build(self): + # Create an embedding layer to map input tokens to vectors + self.embedding_layer = embedding(self.vocab_size,self.embed_size) + # Create a LSTM layer to process the input sequence + self.lstm_layer = LSTM(weight_shape=[self.embed_size,self.embed_size],return_sequence=True) + # Create a self attention layer to attend to the LSTM output + self.self_attention_layer = self_attention(weight_shape=[self.embed_size,self.embed_size],num_heads=self.num_heads) + # Create a linear layer to map the attention output to logits + self.logits_layer = dense([self.embed_size,self.vocab_size]) + # Store the parameters of the layers in a list + self.param = [self.embedding_layer.embeddings] + self.param.extend(self.lstm_layer.param) + self.param.extend(self.self_attention_layer.param) + self.param.extend(self.logits_layer.param) + + + def fp(self,data): + # Perform forward propagation on the input data + x = self.embedding_layer.output(data) # shape: (batch_size, seq_len, embed_size) + x = self.lstm_layer.output(x) # shape: (batch_size, seq_len, embed_size) + x, attention_weights = self.self_attention_layer.output(x,self.num_heads) # shape: (batch_size, seq_len, embed_size) + output = self.logits_layer.output(x) # shape: (batch_size, seq_len, vocab_size) + return output + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) + + + def opt(self,gradient): + # Perform optimization on the parameters using the gradient and the batch count + param=self.optimizer.opt(gradient,self.param,self.bc[0]) + return param \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/parallel/transformer.py b/Note 7.0 documentation/DL/neural network/tensorflow/parallel/transformer.py new file mode 100644 index 00000000..ae052538 --- /dev/null +++ b/Note 7.0 documentation/DL/neural network/tensorflow/parallel/transformer.py @@ -0,0 +1,55 @@ +import tensorflow as tf +from Note.nn.parallel.optimizer import Adam +from Note.nn.layer.transformer import transformer +from Note.nn.layer.dense import dense +from Note.nn.layer.emdedding import embedding +from Note.nn.positional_encoding import positional_encoding + +# Define a positional encoding transformer neural network class +class Transformer: + def __init__(self,vocab_size,embed_size,num_heads,num_layers,max_len): + self.vocab_size=vocab_size + self.embed_size=embed_size + self.num_heads=num_heads + self.num_layers=num_layers + self.max_len=max_len + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.optimizer=Adam() + # Initialize a variable to keep track of the batch count + self.bc=tf.Variable(0,dtype=tf.float32) + + + def build(self): + # Create an embedding layer to map input tokens to vectors + self.embedding_layer = embedding(self.vocab_size,self.embed_size) + # Create a list of transformer layers + self.transformer_layers = [transformer(weight_shape=[self.embed_size, self.embed_size], num_heads=self.num_heads) for _ in range(self.num_layers)] + # Create a linear layer to map the transformer output to logits + self.logits_layer = dense([self.embed_size,self.vocab_size]) + # Store the parameters of the layers in a list + self.param = [self.embedding_layer.embeddings] + for transformer_layer in self.transformer_layers: + self.param.extend(transformer_layer.param) + self.param.extend(self.logits_layer.param) + + + def fp(self,data): + # Perform forward propagation on the input data + x = self.embedding_layer.output(data) # shape: (batch_size, seq_len, embed_size) + x += positional_encoding(self.max_len,self.embed_size) # shape: (batch_size, seq_len, embed_size) + for transformer_layer in self.transformer_layers: + x = transformer_layer.output(x) # shape: (batch_size, seq_len, embed_size) + output = self.logits_layer.output(x) # shape: (batch_size, seq_len, vocab_size) + return output + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) + + + def opt(self,gradient): + # Perform optimization on the parameters using the gradient and the batch count + param=self.optimizer.opt(gradient,self.param,self.bc[0]) + return param \ No newline at end of file From 5328b07c5d2626896f56491c9b995290bd1d32c1 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 27 Jul 2023 13:39:59 +0800 Subject: [PATCH 102/337] Delete Note 7.0 documentation/DL/neural network/tensorflow/process directory --- .../neural network/tensorflow/process/LSTM.py | 46 ------------- .../tensorflow/process/attn_LSTM.py | 53 --------------- .../neural network/tensorflow/process/cnn.py | 59 ---------------- .../tensorflow/process/cnn_lmix.py | 67 ------------------- .../neural network/tensorflow/process/nn.py | 39 ----------- .../neural network/tensorflow/process/nn_.py | 39 ----------- .../tensorflow/process/nn_acc.py | 44 ------------ .../tensorflow/process/nn_attenuate.py | 49 -------------- .../tensorflow/process/nn_clipping.py | 47 ------------- .../tensorflow/process/nn_device.py | 40 ----------- .../tensorflow/process/self_LSTM.py | 54 --------------- .../tensorflow/process/transformer.py | 55 --------------- 12 files changed, 592 deletions(-) delete mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/process/LSTM.py delete mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/process/attn_LSTM.py delete mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/process/cnn.py delete mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/process/cnn_lmix.py delete mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/process/nn.py delete mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/process/nn_.py delete mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/process/nn_acc.py delete mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/process/nn_attenuate.py delete mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/process/nn_clipping.py delete mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/process/nn_device.py delete mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/process/self_LSTM.py delete mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/process/transformer.py diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/process/LSTM.py b/Note 7.0 documentation/DL/neural network/tensorflow/process/LSTM.py deleted file mode 100644 index 93ac8bf0..00000000 --- a/Note 7.0 documentation/DL/neural network/tensorflow/process/LSTM.py +++ /dev/null @@ -1,46 +0,0 @@ -import tensorflow as tf -from Note.nn.layer.LSTM import LSTM -from Note.nn.layer.dense import dense -from Note.nn.process.optimizer import Adam - -# Define a recurrent neural network class with LSTM layers -class lstm: - def __init__(self): - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.optimizer=Adam() - # Initialize a variable to keep track of the batch count - self.bc=tf.Variable(0, dtype=tf.float32) - - - def build(self): - # Create two LSTM layers with 50 hidden units each - self.lstm1=LSTM(weight_shape=(3,50),return_sequence=True) # First LSTM layer that returns the full sequence - self.lstm2=LSTM(weight_shape=(50,50),) # Second LSTM layer that returns the last output only - # Create a dense layer with 10 output units - self.dense=dense([50, 10]) - # Store the parameters of the layers in a list - self.param=[self.lstm1.param, - self.lstm2.param, - self.dense.param, - ] - return - - - def fp(self,data): - # Perform forward propagation on the input data - x=self.lstm1.output(data) # First LSTM layer output - x=self.lstm2.output(x) # Second LSTM layer output - output=self.dense.output(x) # Dense layer output - return output - - - def loss(self, output, labels): - # Compute the loss between the output and the labels - return self.loss_object(labels, output) - - - def opt(self,gradient): - # Perform optimization on the parameters using the gradient and the batch count - param=self.optimizer.opt(gradient,self.param,self.bc[0]) - return param \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/process/attn_LSTM.py b/Note 7.0 documentation/DL/neural network/tensorflow/process/attn_LSTM.py deleted file mode 100644 index 2326ad57..00000000 --- a/Note 7.0 documentation/DL/neural network/tensorflow/process/attn_LSTM.py +++ /dev/null @@ -1,53 +0,0 @@ -import tensorflow as tf -from Note.nn.layer.LSTM import LSTM -from Note.nn.layer.dense import dense -from Note.nn.layer.attention import attention -from Note.nn.process.optimizer import Adam - -# Define a recurrent neural network class with LSTM layers and attention -class attn_lstm: - def __init__(self): - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.optimizer=Adam() - # Initialize a variable to keep track of the batch count - self.bc=tf.Variable(0, dtype=tf.float32) - - - def build(self): - # Create two LSTM layers with 50 hidden units each - self.lstm1=LSTM(weight_shape=(3,50),return_sequence=True) # First LSTM layer that returns the full sequence - self.lstm2=LSTM(weight_shape=(50,50),) # Second LSTM layer that returns the last output only - # Create a dense layer with 10 output units and softmax activation - self.dense=dense([100, 10],activation='softmax') - # Create an attention object with weight shape (50, 50) - self.attn=attention((50,50)) - # Store the parameters of the layers and the attention in a list - self.param=[self.lstm1.param, - self.lstm2.param, - self.dense.param, - self.attn.param] # Add the attention parameters to the list - return - - - def fp(self,data): - # Perform forward propagation on the input data - x=self.lstm1.output(data) # First LSTM layer output, shape (B, N, 50) - y=self.lstm2.output(x) # Second LSTM layer output, shape (B, 50) - # Use the attention object to compute the context vector from x and y, shape (B, 50) - context_vector,_,_=self.attn.output(x,y) - # Concatenate the context vector and the second LSTM output, shape (B, 100) - z=tf.concat([context_vector,y],axis=-1) - output=self.dense.output(z) # Dense layer output with softmax activation, shape (B, 10) - return output - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) - - - def opt(self,gradient): - # Perform optimization on the parameters using the gradient and the batch count - param=self.optimizer.opt(gradient,self.param,self.bc[0]) - return param \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/process/cnn.py b/Note 7.0 documentation/DL/neural network/tensorflow/process/cnn.py deleted file mode 100644 index 49892ccc..00000000 --- a/Note 7.0 documentation/DL/neural network/tensorflow/process/cnn.py +++ /dev/null @@ -1,59 +0,0 @@ -import tensorflow as tf -from Note.nn.layer.conv2d import conv2d -from Note.nn.layer.dense import dense -from Note.nn.layer.flatten import flatten -from Note.nn.process.optimizer import Adam - -# Define a convolutional neural network class -class cnn: - def __init__(self): - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.optimizer=Adam() - # Initialize a variable to keep track of the batch count - self.bc=tf.Variable(0,dtype=tf.float32) - - - def build(self): - # Create three convolutional layers with relu activations - self.conv1=conv2d(weight_shape=(3,3,3,32),activation='relu') - self.conv2=conv2d(weight_shape=(3,3,32,64),activation='relu') - self.conv3=conv2d(weight_shape=(3,3,64,64),activation='relu') - # Create two dense layers with relu and linear activations - self.dense1=dense([64*4*4,64],activation='relu') - self.dense2=dense([64,10]) - # Store the parameters of the layers in a list - self.param = [self.conv1.weight, - self.conv2.weight, - self.conv3.weight, - self.dense1.weight, - self.dense1.bias, - self.dense2.weight, - self.dense2.bias] - return - - - def fp(self,data): - # Perform forward propagation on the input data - x=tf.nn.conv2d(data,self.conv1.weight,strides=(1,1),padding='SAME') # First convolutional layer - x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # First max pooling layer - x=tf.nn.conv2d(x,self.conv2.weight,strides=(1,1),padding='SAME') # Second convolutional layer - x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # Second max pooling layer - x=tf.nn.conv2d(x,self.conv3.weight,strides=(1,1),padding='SAME') # Third convolutional layer - x=flatten(x) # Flatten the output to a vector - x=tf.nn.relu(tf.matmul(x,self.dense1.weight)+self.dense1.bias) # First dense layer with relu activation - x=tf.nn.dropout(x,rate=0.5) # Apply dropout to prevent overfitting - output=tf.matmul(x,self.dense2.weight)+self.dense2.bias # Output layer with linear activation - output=tf.nn.dropout(output,rate=0.5) # Apply dropout to prevent overfitting - return output - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) - - - def opt(self,gradient): - # Perform optimization on the parameters using the gradient and the batch count - param=self.optimizer.opt(gradient,self.param,self.bc[0]) - return param \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/process/cnn_lmix.py b/Note 7.0 documentation/DL/neural network/tensorflow/process/cnn_lmix.py deleted file mode 100644 index c1fa4cc9..00000000 --- a/Note 7.0 documentation/DL/neural network/tensorflow/process/cnn_lmix.py +++ /dev/null @@ -1,67 +0,0 @@ -import tensorflow as tf -from Note.nn.layer.conv2d import conv2d -from Note.nn.layer.dense import dense -from Note.nn.layer.flatten import flatten -from Note.nn.process.optimizer import Adam -from Note.nn.layer.LMix import lmix - -# Define a convolutional neural network class with mixup augmentation -class cnn: - def __init__(self): - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.optimizer=Adam() - # Initialize the alpha parameter for mixup - self.alpha=1.0 - # Initialize a variable to keep track of the batch count - self.bc=tf.Variable(0,dtype=tf.float32) - - - def build(self): - # Create three convolutional layers with relu activations - self.conv1=conv2d(weight_shape=(3,3,3,32),activation='relu') - self.conv2=conv2d(weight_shape=(3,3,32,64),activation='relu') - self.conv3=conv2d(weight_shape=(3,3,64,64),activation='relu') - # Create two dense layers with relu and linear activations - self.dense1=dense([64*4*4,64],activation='relu') - self.dense2=dense([64,10]) - # Store the parameters of the layers in a list - self.param = [self.conv1.weight, - self.conv2.weight, - self.conv3.weight, - self.dense1.weight, - self.dense1.bias, - self.dense2.weight, - self.dense2.bias] - return - - - def data_func(self,data_batch,labels_batch): - # Apply mixup augmentation on the input data and labels - return lmix(data_batch,labels_batch,self.alpha) - - - def fp(self,data): - # Perform forward propagation on the input data - x=tf.nn.conv2d(data,self.conv1.weight,strides=(1,1),padding='SAME') # First convolutional layer - x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # First max pooling layer - x=tf.nn.conv2d(x,self.conv2.weight,strides=(1,1),padding='SAME') # Second convolutional layer - x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # Second max pooling layer - x=tf.nn.conv2d(x,self.conv3.weight,strides=(1,1),padding='SAME') # Third convolutional layer - x=flatten(x) # Flatten the output to a vector - x=tf.nn.relu(tf.matmul(x,self.dense1.weight)+self.dense1.bias) # First dense layer with relu activation - x=tf.nn.dropout(x,rate=0.5) # Apply dropout to prevent overfitting - output=tf.matmul(x,self.dense2.weight)+self.dense2.bias # Output layer with linear activation - output=tf.nn.dropout(output,rate=0.5) # Apply dropout to prevent overfitting - return output - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) - - - def opt(self,gradient): - # Perform optimization on the parameters using the gradient and the batch count - param=self.optimizer.opt(gradient,self.param,self.bc[0]) - return param \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/process/nn.py b/Note 7.0 documentation/DL/neural network/tensorflow/process/nn.py deleted file mode 100644 index 70f2a637..00000000 --- a/Note 7.0 documentation/DL/neural network/tensorflow/process/nn.py +++ /dev/null @@ -1,39 +0,0 @@ -import tensorflow as tf -import Note.nn.layer.dense as d -from Note.nn.process.optimizer import Momentum -from Note.nn.layer.flatten import flatten - -# Define a neural network class -class nn: - def __init__(self): - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.optimizer=Momentum(0.07,0.7) - - - def build(self): - # Create two dense layers with relu and linear activations - self.layer1=d.dense([784,128],activation='relu') - self.layer2=d.dense([128,10]) - # Store the parameters of the layers in a list - self.param=[self.layer1.param,self.layer2.param] - return - - - def fp(self,data): - # Perform forward propagation on the input data - data=flatten(data) - output1=self.layer1.output(data) - output2=self.layer2.output(output1) - return output2 - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) - - - def opt(self,gradient): - # Perform optimization on the parameters using the gradient - param=self.optimizer.opt(gradient,self.param) - return param \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/process/nn_.py b/Note 7.0 documentation/DL/neural network/tensorflow/process/nn_.py deleted file mode 100644 index 3f50c1e3..00000000 --- a/Note 7.0 documentation/DL/neural network/tensorflow/process/nn_.py +++ /dev/null @@ -1,39 +0,0 @@ -import tensorflow as tf -from Note.nn.process.optimizer import Momentum -from Note.nn.layer.flatten import flatten - -# Define a neural network class -class nn: - def __init__(self): - # Initialize the weights and biases of three layers with random values - self.weight1=tf.Variable(tf.random.normal([784,64],dtype='float64')) - self.bias1=tf.Variable(tf.random.normal([64],dtype='float64')) - self.weight2=tf.Variable(tf.random.normal([64,64],dtype='float64')) - self.bias2=tf.Variable(tf.random.normal([64],dtype='float64')) - self.weight3=tf.Variable(tf.random.normal([64,10],dtype='float64')) - self.bias3=tf.Variable(tf.random.normal([10],dtype='float64')) - # Store the parameters of the layers in a list - self.param=[self.weight1,self.weight2,self.weight3,self.bias1,self.bias2,self.bias3] - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.optimizer=Momentum(0.07,0.7) - - - def fp(self,data): - # Perform forward propagation on the input data - data=flatten(data) - layer1=tf.nn.relu(tf.matmul(data,self.weight1)+self.bias1) # First layer with relu activation - layer2=tf.nn.relu(tf.matmul(layer1,self.weight2)+self.bias2) # Second layer with relu activation - output=tf.matmul(layer2,self.weight3)+self.bias3 # Output layer with linear activation - return output - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) - - - def opt(self,gradient): - # Perform optimization on the parameters using the gradient - param=self.optimizer.opt(gradient,self.param) - return param \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/process/nn_acc.py b/Note 7.0 documentation/DL/neural network/tensorflow/process/nn_acc.py deleted file mode 100644 index 11f29cda..00000000 --- a/Note 7.0 documentation/DL/neural network/tensorflow/process/nn_acc.py +++ /dev/null @@ -1,44 +0,0 @@ -import tensorflow as tf -import Note.nn.layer.dense as d -from Note.nn.process.optimizer import Momentum -from Note.nn.layer.flatten import flatten -from Note.nn.accuracy import sparse_categorical_accuracy - -# Define a neural network class -class nn: - def __init__(self): - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.optimizer=Momentum(0.07,0.7) - - - def build(self): - # Create two dense layers with relu and linear activations - self.layer1=d.dense([784,128],activation='relu') - self.layer2=d.dense([128,10]) - # Store the parameters of the layers in a list - self.param=[self.layer1.param,self.layer2.param] - return - - - def fp(self,data): - # Perform forward propagation on the input data - data=flatten(data) - output1=self.layer1.output(data) - output2=self.layer2.output(output1) - return output2 - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) - - - def accuracy(self,output,labels): #accuracy function,kernel uses it to calculate accuracy. - return sparse_categorical_accuracy(labels,output) - - - def opt(self,gradient): - # Perform optimization on the parameters using the gradient - param=self.optimizer.opt(gradient,self.param) - return param \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/process/nn_attenuate.py b/Note 7.0 documentation/DL/neural network/tensorflow/process/nn_attenuate.py deleted file mode 100644 index 890b3335..00000000 --- a/Note 7.0 documentation/DL/neural network/tensorflow/process/nn_attenuate.py +++ /dev/null @@ -1,49 +0,0 @@ -import tensorflow as tf -import Note.nn.layer.dense as d -from Note.nn.process.optimizer import Momentum -from Note.nn.layer.flatten import flatten - -# Define a neural network class with gradient attenuation -class nn: - def __init__(self): - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.optimizer=Momentum(0.07,0.7) - # Initialize a variable to keep track of the number of optimization steps - self.opt_counter=tf.Variable(tf.zeros(7,dtype=tf.float32)) - - - def build(self): - # Create two dense layers with relu and linear activations - self.layer1=d.dense([784,128],activation='relu') - self.layer2=d.dense([128,10]) - # Store the parameters of the layers in a list - self.param=[self.layer1.param,self.layer2.param] - return - - - def fp(self,data): - # Perform forward propagation on the input data - data=flatten(data) - output1=self.layer1.output(data) - output2=self.layer2.output(output1) - return output2 - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) - - - def attenuate(self,gradient,oc,p): #gradient attenuation function,kernel uses it to calculate attenuation coefficient. - # Apply an exponential decay to the gradient based on the optimization counter - ac=0.9**oc[p] #ac:attenuation coefficient - for i in range(len(gradient)): #oc:optimization counter - gradient[i]=ac*gradient[i] #p:process number - return gradient - - - def opt(self,gradient): - # Perform optimization on the parameters using the gradient - param=self.optimizer.opt(gradient,self.param) - return param \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/process/nn_clipping.py b/Note 7.0 documentation/DL/neural network/tensorflow/process/nn_clipping.py deleted file mode 100644 index 32779f84..00000000 --- a/Note 7.0 documentation/DL/neural network/tensorflow/process/nn_clipping.py +++ /dev/null @@ -1,47 +0,0 @@ -import tensorflow as tf -import Note.nn.layer.dense as d -from Note.nn.process.optimizer import Momentum -from Note.nn.layer.flatten import flatten -from Note.nn.gradient_clipping import gradient_clipping - -# Define a neural network class -class nn: - def __init__(self): - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.optimizer=Momentum(0.07,0.7) - - - def build(self): - # Create two dense layers with relu and linear activations - self.layer1=d.dense([784,128],activation='relu') - self.layer2=d.dense([128,10]) - # Store the parameters of the layers in a list - self.param=[self.layer1.param,self.layer2.param] - return - - - def fp(self,data): - # Perform forward propagation on the input data - data=flatten(data) - output1=self.layer1.output(data) - output2=self.layer2.output(output1) - return output2 - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) - - - def gradient(self,tape,loss): - # Compute the gradients of the loss with respect to the parameters - grads=tape.gradient(loss,self.param) - # Clip the gradients by value to avoid exploding gradients - return gradient_clipping(grads,'value',1.0) - - - def opt(self,gradient): - # Perform optimization on the parameters using the gradient - param=self.optimizer.opt(gradient,self.param) - return param \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/process/nn_device.py b/Note 7.0 documentation/DL/neural network/tensorflow/process/nn_device.py deleted file mode 100644 index 1b37778c..00000000 --- a/Note 7.0 documentation/DL/neural network/tensorflow/process/nn_device.py +++ /dev/null @@ -1,40 +0,0 @@ -import tensorflow as tf # import TensorFlow library -import Note.nn.layer.dense as d # import Note's dense layer module -from Note.nn.layer.flatten import flatten # import Note's flatten layer function -from Note.nn.process.optimizer import Momentum # import Note's momentum optimizer module -from Note.nn.process.assign_device import assign_device # import the function to assign device according to the process index and the device type - - -class nn: # A neural network class example, allocate device for multiple threads - def __init__(self): # initialize the network - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) # loss object, kernel uses it to calculate loss. Here we use sparse categorical crossentropy with logits as output - self.optimizer=Momentum(0.07,0.7) # optimizer, kernel uses it to optimize. Here we use a custom momentum optimizer with learning rate 0.07 and momentum 0.7 - self.info='example' # some information about the network - - - def build(self): # build function, kernel uses it to create the network layers - # Create two dense layers with relu and linear activations - self.layer1=d.dense([784,128],activation='relu') # the first layer with 784 input units and 128 output units and ReLU activation - self.layer2=d.dense([128,10]) # the second layer with 128 input units and 10 output units and linear activation - # Store the parameters of the layers in a list - self.param=[self.layer1.param,self.layer2.param] # parameter list of both layers, kernel uses it list for backpropagation - return - - - def fp(self,data,p): # forward propagation function, kernel uses it for forward propagation - with tf.device(assign_device(p,'GPU')): # assign the device according to the process index p - data=flatten(data) # flatten the data to a vector of 784 elements - output1=self.layer1.output(data) # pass the data through the first layer and get the output - output2=self.layer2.output(output1) # pass the output of the first layer through the second layer and get the final output logits - return output2 - - - def loss(self,output,labels,p): # loss function, kernel uses it to calculate loss - with tf.device(assign_device(p,'GPU')): # assign the device according to the process index p - return self.loss_object(labels,output) # return the mean softmax cross entropy loss between labels and output - - - def opt(self,gradient): # optimization function, kernel uses it to optimize parameter - # Perform optimization on the parameters using the gradient - param=self.optimizer.opt(gradient,self.param) # apply the Note's momentum optimizer to update the parameters using the gradient - return param diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/process/self_LSTM.py b/Note 7.0 documentation/DL/neural network/tensorflow/process/self_LSTM.py deleted file mode 100644 index a25f4043..00000000 --- a/Note 7.0 documentation/DL/neural network/tensorflow/process/self_LSTM.py +++ /dev/null @@ -1,54 +0,0 @@ -import tensorflow as tf -from Note.nn.process.optimizer import Adam -from Note.nn.layer.LSTM import LSTM -from Note.nn.layer.self_attention import self_attention -from Note.nn.layer.dense import dense -from Note.nn.layer.emdedding import embedding - -# Define a LSTM self attention neural network class -class self_LSTM: - def __init__(self,vocab_size,embed_size,num_heads,num_layers): - self.vocab_size=vocab_size - self.embed_size=embed_size - self.num_heads=num_heads - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.optimizer=Adam() - # Initialize a variable to keep track of the batch count - self.bc=tf.Variable(0,dtype=tf.float32) - - - def build(self): - # Create an embedding layer to map input tokens to vectors - self.embedding_layer = embedding(self.vocab_size,self.embed_size) - # Create a LSTM layer to process the input sequence - self.lstm_layer = LSTM(weight_shape=[self.embed_size,self.embed_size],return_sequence=True) - # Create a self attention layer to attend to the LSTM output - self.self_attention_layer = self_attention(weight_shape=[self.embed_size,self.embed_size],num_heads=self.num_heads) - # Create a linear layer to map the attention output to logits - self.logits_layer = dense([self.embed_size,self.vocab_size]) - # Store the parameters of the layers in a list - self.param = [self.embedding_layer.embeddings] - self.param.extend(self.lstm_layer.param) - self.param.extend(self.self_attention_layer.param) - self.param.extend(self.logits_layer.param) - - - def fp(self,data): - # Perform forward propagation on the input data - x = self.embedding_layer.output(data) # shape: (batch_size, seq_len, embed_size) - x = self.lstm_layer.output(x) # shape: (batch_size, seq_len, embed_size) - x, attention_weights = self.self_attention_layer.output(x,self.num_heads) # shape: (batch_size, seq_len, embed_size) - output = self.logits_layer.output(x) # shape: (batch_size, seq_len, vocab_size) - return output - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) - - - def opt(self,gradient): - # Perform optimization on the parameters using the gradient and the batch count - param=self.optimizer.opt(gradient,self.param,self.bc[0]) - return param \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/process/transformer.py b/Note 7.0 documentation/DL/neural network/tensorflow/process/transformer.py deleted file mode 100644 index a77978a6..00000000 --- a/Note 7.0 documentation/DL/neural network/tensorflow/process/transformer.py +++ /dev/null @@ -1,55 +0,0 @@ -import tensorflow as tf -from Note.nn.process.optimizer import Adam -from Note.nn.layer.transformer import transformer -from Note.nn.layer.dense import dense -from Note.nn.layer.emdedding import embedding -from Note.nn.positional_encoding import positional_encoding - -# Define a positional encoding transformer neural network class -class Transformer: - def __init__(self,vocab_size,embed_size,num_heads,num_layers,max_len): - self.vocab_size=vocab_size - self.embed_size=embed_size - self.num_heads=num_heads - self.num_layers=num_layers - self.max_len=max_len - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.optimizer=Adam() - # Initialize a variable to keep track of the batch count - self.bc=tf.Variable(0,dtype=tf.float32) - - - def build(self): - # Create an embedding layer to map input tokens to vectors - self.embedding_layer = embedding(self.vocab_size,self.embed_size) - # Create a list of transformer layers - self.transformer_layers = [transformer(weight_shape=[self.embed_size, self.embed_size], num_heads=self.num_heads) for _ in range(self.num_layers)] - # Create a linear layer to map the transformer output to logits - self.logits_layer = dense([self.embed_size,self.vocab_size]) - # Store the parameters of the layers in a list - self.param = [self.embedding_layer.embeddings] - for transformer_layer in self.transformer_layers: - self.param.extend(transformer_layer.param) - self.param.extend(self.logits_layer.param) - - - def fp(self,data): - # Perform forward propagation on the input data - x = self.embedding_layer.output(data) # shape: (batch_size, seq_len, embed_size) - x += positional_encoding(self.max_len,self.embed_size) # shape: (batch_size, seq_len, embed_size) - for transformer_layer in self.transformer_layers: - x = transformer_layer.output(x) # shape: (batch_size, seq_len, embed_size) - output = self.logits_layer.output(x) # shape: (batch_size, seq_len, vocab_size) - return output - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) - - - def opt(self,gradient): - # Perform optimization on the parameters using the gradient and the batch count - param=self.optimizer.opt(gradient,self.param,self.bc[0]) - return param \ No newline at end of file From 29553c6fff6bb422aab1a5263942ef3b9e7b66dd Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 27 Jul 2023 13:40:17 +0800 Subject: [PATCH 103/337] Delete bert.py --- .../DL/neural network/tensorflow/bert.py | 167 ------------------ 1 file changed, 167 deletions(-) delete mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/bert.py diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/bert.py b/Note 7.0 documentation/DL/neural network/tensorflow/bert.py deleted file mode 100644 index 9715477f..00000000 --- a/Note 7.0 documentation/DL/neural network/tensorflow/bert.py +++ /dev/null @@ -1,167 +0,0 @@ -import tensorflow as tf -import tensorflow_hub as hub - - -class bert: - def __init__(self): - text_input=tf.keras.layers.Input(shape=(),dtype=tf.string,name='text') - preprocessing_layer=hub.KerasLayer(tfhub_handle_preprocess,name='preprocessing') - encoder_inputs=preprocessing_layer(text_input) - encoder=hub.KerasLayer(tfhub_handle_encoder,trainable=True,name='BERT_encoder') - outputs=encoder(encoder_inputs) - net=outputs['pooled_output'] - net=tf.keras.layers.Dropout(0.1)(net) - net=tf.keras.layers.Dense(1,activation=None,name='classifier')(net) - self.model=tf.keras.Model(text_input,net) - self.param=self.model.weights - self.opt=tf.keras.optimizers.Adam() - - - def fp(self,data): - return self.model(data) - - - def loss(self,output,labels): - bce=tf.keras.losses.BinaryCrossentropy(from_logits=True) - return bce(output,labels) - - -bert_model_name = 'small_bert/bert_en_uncased_L-4_H-512_A-8' -map_name_to_handle = { - 'bert_en_uncased_L-12_H-768_A-12': - 'https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/3', - 'bert_en_cased_L-12_H-768_A-12': - 'https://tfhub.dev/tensorflow/bert_en_cased_L-12_H-768_A-12/3', - 'bert_multi_cased_L-12_H-768_A-12': - 'https://tfhub.dev/tensorflow/bert_multi_cased_L-12_H-768_A-12/3', - 'small_bert/bert_en_uncased_L-2_H-128_A-2': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-2_H-128_A-2/1', - 'small_bert/bert_en_uncased_L-2_H-256_A-4': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-2_H-256_A-4/1', - 'small_bert/bert_en_uncased_L-2_H-512_A-8': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-2_H-512_A-8/1', - 'small_bert/bert_en_uncased_L-2_H-768_A-12': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-2_H-768_A-12/1', - 'small_bert/bert_en_uncased_L-4_H-128_A-2': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-128_A-2/1', - 'small_bert/bert_en_uncased_L-4_H-256_A-4': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-256_A-4/1', - 'small_bert/bert_en_uncased_L-4_H-512_A-8': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-512_A-8/1', - 'small_bert/bert_en_uncased_L-4_H-768_A-12': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-768_A-12/1', - 'small_bert/bert_en_uncased_L-6_H-128_A-2': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-6_H-128_A-2/1', - 'small_bert/bert_en_uncased_L-6_H-256_A-4': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-6_H-256_A-4/1', - 'small_bert/bert_en_uncased_L-6_H-512_A-8': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-6_H-512_A-8/1', - 'small_bert/bert_en_uncased_L-6_H-768_A-12': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-6_H-768_A-12/1', - 'small_bert/bert_en_uncased_L-8_H-128_A-2': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-8_H-128_A-2/1', - 'small_bert/bert_en_uncased_L-8_H-256_A-4': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-8_H-256_A-4/1', - 'small_bert/bert_en_uncased_L-8_H-512_A-8': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-8_H-512_A-8/1', - 'small_bert/bert_en_uncased_L-8_H-768_A-12': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-8_H-768_A-12/1', - 'small_bert/bert_en_uncased_L-10_H-128_A-2': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-10_H-128_A-2/1', - 'small_bert/bert_en_uncased_L-10_H-256_A-4': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-10_H-256_A-4/1', - 'small_bert/bert_en_uncased_L-10_H-512_A-8': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-10_H-512_A-8/1', - 'small_bert/bert_en_uncased_L-10_H-768_A-12': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-10_H-768_A-12/1', - 'small_bert/bert_en_uncased_L-12_H-128_A-2': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-12_H-128_A-2/1', - 'small_bert/bert_en_uncased_L-12_H-256_A-4': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-12_H-256_A-4/1', - 'small_bert/bert_en_uncased_L-12_H-512_A-8': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-12_H-512_A-8/1', - 'small_bert/bert_en_uncased_L-12_H-768_A-12': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-12_H-768_A-12/1', - 'albert_en_base': - 'https://tfhub.dev/tensorflow/albert_en_base/2', - 'electra_small': - 'https://tfhub.dev/google/electra_small/2', - 'electra_base': - 'https://tfhub.dev/google/electra_base/2', - 'experts_pubmed': - 'https://tfhub.dev/google/experts/bert/pubmed/2', - 'experts_wiki_books': - 'https://tfhub.dev/google/experts/bert/wiki_books/2', - 'talking-heads_base': - 'https://tfhub.dev/tensorflow/talkheads_ggelu_bert_en_base/1', -} -map_model_to_preprocess = { - 'bert_en_uncased_L-12_H-768_A-12': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'bert_en_cased_L-12_H-768_A-12': - 'https://tfhub.dev/tensorflow/bert_en_cased_preprocess/3', - 'small_bert/bert_en_uncased_L-2_H-128_A-2': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-2_H-256_A-4': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-2_H-512_A-8': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-2_H-768_A-12': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-4_H-128_A-2': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-4_H-256_A-4': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-4_H-512_A-8': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-4_H-768_A-12': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-6_H-128_A-2': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-6_H-256_A-4': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-6_H-512_A-8': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-6_H-768_A-12': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-8_H-128_A-2': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-8_H-256_A-4': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-8_H-512_A-8': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-8_H-768_A-12': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-10_H-128_A-2': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-10_H-256_A-4': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-10_H-512_A-8': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-10_H-768_A-12': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-12_H-128_A-2': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-12_H-256_A-4': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-12_H-512_A-8': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-12_H-768_A-12': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'bert_multi_cased_L-12_H-768_A-12': - 'https://tfhub.dev/tensorflow/bert_multi_cased_preprocess/3', - 'albert_en_base': - 'https://tfhub.dev/tensorflow/albert_en_preprocess/3', - 'electra_small': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'electra_base': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'experts_pubmed': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'experts_wiki_books': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'talking-heads_base': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', -} -tfhub_handle_encoder=map_name_to_handle[bert_model_name] -tfhub_handle_preprocess=map_model_to_preprocess[bert_model_name] \ No newline at end of file From 92fb57068c8b166cf6aaadf39f398c0a81ee42ed Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 27 Jul 2023 13:40:29 +0800 Subject: [PATCH 104/337] Delete cnn.py --- .../DL/neural network/tensorflow/cnn.py | 27 ------------------- 1 file changed, 27 deletions(-) delete mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/cnn.py diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/cnn.py b/Note 7.0 documentation/DL/neural network/tensorflow/cnn.py deleted file mode 100644 index bd7ad1bf..00000000 --- a/Note 7.0 documentation/DL/neural network/tensorflow/cnn.py +++ /dev/null @@ -1,27 +0,0 @@ -import tensorflow as tf -from tensorflow.keras import layers,models - - -class cnn: - def __init__(self): - self.model=models.Sequential() - self.model.add(layers.Conv2D(32,(3,3),activation='relu',input_shape=(32,32,3))) - self.model.add(layers.MaxPooling2D((2,2))) - self.model.add(layers.Conv2D(64,(3,3),activation='relu')) - self.model.add(layers.MaxPooling2D((2,2))) - self.model.add(layers.Conv2D(64,(3,3),activation='relu')) - self.model.add(layers.Flatten()) - self.model.add(layers.Dense(64,activation='relu')) - self.model.add(layers.Dense(10)) - self.param=self.model.weights - self.opt=tf.keras.optimizers.Adam() - - - def fp(self,data): - output=self.model(data) - return output - - - def loss(self,output,labels): - loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - return loss(labels,output) \ No newline at end of file From 16c5ccb5f2cd0f7293b1c218c7fd95c6435b0a02 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 27 Jul 2023 13:40:41 +0800 Subject: [PATCH 105/337] Delete lstm.py --- .../DL/neural network/tensorflow/lstm.py | 26 ------------------- 1 file changed, 26 deletions(-) delete mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/lstm.py diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/lstm.py b/Note 7.0 documentation/DL/neural network/tensorflow/lstm.py deleted file mode 100644 index 29b9b396..00000000 --- a/Note 7.0 documentation/DL/neural network/tensorflow/lstm.py +++ /dev/null @@ -1,26 +0,0 @@ -import tensorflow as tf - - -class lstm: - def __init__(self,encoder): - self.model=tf.keras.Sequential([ - encoder, - tf.keras.layers.Embedding(len(encoder.get_vocabulary()), 64, mask_zero=True), - tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(64, return_sequences=True)), - tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(32)), - tf.keras.layers.Dense(64, activation='relu'), - tf.keras.layers.Dropout(0.5), - tf.keras.layers.Dense(1) - ]) - self.param=self.model.weights[1:] - self.opt=tf.keras.optimizers.Adam() - - - def fp(self,data): - output=self.model(data) - return output - - - def loss(self,output,labels): - bce=tf.keras.losses.BinaryCrossentropy(from_logits=True) - return bce(labels,output) \ No newline at end of file From 3b2af8435b078af9fee2fc12737400527d0aec59 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 27 Jul 2023 13:40:52 +0800 Subject: [PATCH 106/337] Delete nn.py --- .../DL/neural network/tensorflow/nn.py | 23 ------------------- 1 file changed, 23 deletions(-) delete mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/nn.py diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/nn.py b/Note 7.0 documentation/DL/neural network/tensorflow/nn.py deleted file mode 100644 index b4f03a92..00000000 --- a/Note 7.0 documentation/DL/neural network/tensorflow/nn.py +++ /dev/null @@ -1,23 +0,0 @@ -import tensorflow as tf - - -class nn: - def __init__(self): - self.model=tf.keras.models.Sequential([ - tf.keras.layers.Flatten(input_shape=(28, 28)), - tf.keras.layers.Dense(128,activation='relu'), - tf.keras.layers.Dropout(0.2), - tf.keras.layers.Dense(10) - ]) - self.param=self.model.weights #parameter list,kernel uses it list for backpropagation. - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.opt=tf.keras.optimizers.Adam() #optimizer,kernel uses it to optimize. - - - def fp(self,data): #forward propagation function,kernel uses it for forward propagation. - output=self.model(data) - return output - - - def loss(self,output,labels): #loss functino,kernel uses it to calculate loss. - return self.loss_object(labels,output) \ No newline at end of file From 3577cc1efd857fafd318af8d0cb484233d3c661f Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 27 Jul 2023 13:41:02 +0800 Subject: [PATCH 107/337] Delete nn_acc.py --- .../DL/neural network/tensorflow/nn_acc.py | 28 ------------------- 1 file changed, 28 deletions(-) delete mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/nn_acc.py diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/nn_acc.py b/Note 7.0 documentation/DL/neural network/tensorflow/nn_acc.py deleted file mode 100644 index 0a726603..00000000 --- a/Note 7.0 documentation/DL/neural network/tensorflow/nn_acc.py +++ /dev/null @@ -1,28 +0,0 @@ -import tensorflow as tf - -#An example with accuracy function. -class nn: - def __init__(self): - self.model=tf.keras.models.Sequential([ - tf.keras.layers.Flatten(input_shape=(28, 28)), - tf.keras.layers.Dense(128,activation='relu'), - tf.keras.layers.Dropout(0.2), - tf.keras.layers.Dense(10) - ]) - self.param=self.model.weights #parameter list,kernel uses it list for backpropagation. - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.train_accuracy=tf.keras.metrics.SparseCategoricalAccuracy(name='train_accuracy') - self.opt=tf.keras.optimizers.Adam() #optimizer,kernel uses it to optimize. - - - def fp(self,data): #forward propagation function,kernel uses it for forward propagation. - output=self.model(data) - return output - - - def loss(self,output,labels): #loss functino,kernel uses it to calculate loss. - return self.loss_object(labels,output) - - - def accuracy(self,output,labels): #accuracy function,kernel uses it to calculate accuracy. - return self.train_accuracy(labels,output) \ No newline at end of file From 3c5cca8180031a4b515c7f1fde4a9bee81c2ea43 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 27 Jul 2023 13:41:12 +0800 Subject: [PATCH 108/337] Delete nn_ol.py --- .../DL/neural network/tensorflow/nn_ol.py | 37 ------------------- 1 file changed, 37 deletions(-) delete mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/nn_ol.py diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/nn_ol.py b/Note 7.0 documentation/DL/neural network/tensorflow/nn_ol.py deleted file mode 100644 index 7bacada0..00000000 --- a/Note 7.0 documentation/DL/neural network/tensorflow/nn_ol.py +++ /dev/null @@ -1,37 +0,0 @@ -import tensorflow as tf # import TensorFlow library -import numpy as np # import NumPy library - -# This is an example of online training. -class nn: # define a class for the neural network - def __init__(self,data,labels): # initialize the network with data and labels - self.train_data=data # store the training data - self.train_labels=labels # store the training labels - self.model=tf.keras.models.Sequential([ # create a sequential model with four layers - tf.keras.layers.Flatten(input_shape=(28, 28)), # flatten the input to a vector of 28*28 elements - tf.keras.layers.Dense(128,activation='relu'), # add a dense layer with 128 units and ReLU activation - tf.keras.layers.Dropout(0.2), # add a dropout layer with 0.2 dropout rate to prevent overfitting - tf.keras.layers.Dense(10) # add a dense layer with 10 units for the output logits - ]) - self.param=self.model.weights # parameter list, kernel uses it list for backpropagation - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) # loss object, kernel uses it to calculate loss. Here we use sparse categorical crossentropy with logits as output - self.opt=tf.keras.optimizers.Adam() # optimizer, kernel uses it to optimize. Here we use Adam optimizer - self.train_loss_list=[] # a list to store the training loss values - self.counter=0 # counter to keep track of the number of online updates - self.max_length=1000 # maximum length of train_loss_list - - - def online(self): # This is simulative online function, kernel uses it to get online data and labels - index=np.random.choice(60000,size=[32]) # randomly sample 32 indices from 0 to 59999 - if self.counter==10000: # if the counter reaches 10000, stop the online training - return 'stop' - else: # otherwise, return the data and labels corresponding to the sampled indices - return [self.train_data[index],self.train_labels[index]] - - - def fp(self,data): # forward propagation function, kernel uses it for forward propagation - output=self.model(data) # pass the data through the model and get the output logits - return output - - - def loss(self,output,labels): # loss function, kernel uses it to calculate loss - return self.loss_object(labels,output) # return the sparse categorical crossentropy loss between labels and output From 668193a1fd4efc6fe515c8fed0b404944f526888 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 27 Jul 2023 13:44:58 +0800 Subject: [PATCH 109/337] Add files via upload --- .../neural network/pytorch/non-parallel/nn.py | 53 +++++++++++++++++++ .../DL/neural network/pytorch/parallel/nn.py | 41 ++++++++++++++ .../pytorch/parallel/nn_device.py | 45 ++++++++++++++++ 3 files changed, 139 insertions(+) create mode 100644 Note 7.0 documentation/DL/neural network/pytorch/non-parallel/nn.py create mode 100644 Note 7.0 documentation/DL/neural network/pytorch/parallel/nn.py create mode 100644 Note 7.0 documentation/DL/neural network/pytorch/parallel/nn_device.py diff --git a/Note 7.0 documentation/DL/neural network/pytorch/non-parallel/nn.py b/Note 7.0 documentation/DL/neural network/pytorch/non-parallel/nn.py new file mode 100644 index 00000000..1faec0fc --- /dev/null +++ b/Note 7.0 documentation/DL/neural network/pytorch/non-parallel/nn.py @@ -0,0 +1,53 @@ +import torch +from torch import nn + + +class NeuralNetwork(nn.Module): + def __init__(self): + super().__init__() + self.flatten=nn.Flatten() + self.linear_relu_stack=nn.Sequential( + nn.Linear(28*28, 512), + nn.ReLU(), + nn.Linear(512, 512), + nn.ReLU(), + nn.Linear(512, 10) + ) + + + def forward(self,x): + x=self.flatten(x) + logits=self.linear_relu_stack(x) + return logits + + +class neuralnetwork: + def __init__(self): + if torch.cuda.is_available(): + self.device=torch.device('cuda') + else: + self.device=torch.device('cpu') + self.model=NeuralNetwork().to(self.device) + self.loss_fn=nn.CrossEntropyLoss() + self.opt=torch.optim.SGD(self.model.parameters(),lr=1e-3) #optimizer,kernel uses it to optimize. + + + def fp(self,x): #forward propagation function,kernel uses it for forward propagation. + pred=self.model(x.to(self.device)) + return pred + + + def loss(self,output,labels): #loss functino,kernel uses it to calculate loss. + loss=self.loss_fn(output,labels.to(self.device)) + return loss + + + def backward(self,loss): #backward function,kernel uses it for backpropagation. + self.optim.zero_grad() + loss.backward() + return + + + def opt(self): #opt function,kernel uses it to optimize. + self.optim.step() + return \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/pytorch/parallel/nn.py b/Note 7.0 documentation/DL/neural network/pytorch/parallel/nn.py new file mode 100644 index 00000000..d9b05567 --- /dev/null +++ b/Note 7.0 documentation/DL/neural network/pytorch/parallel/nn.py @@ -0,0 +1,41 @@ +import torch # import the PyTorch library +from torch import nn # import the neural network module from PyTorch + +class NeuralNetwork(nn.Module): # define a class for the neural network model + def __init__(self): # define the constructor method + super().__init__() # call the parent class constructor + self.flatten = nn.Flatten() # define a layer to flatten the input + self.linear_relu_stack=nn.Sequential( # define a stack of linear and relu layers + nn.Linear(28*28,512), # a linear layer with 28*28 input features and 512 output features + nn.ReLU(), # a relu activation function + nn.Linear(512,512), # another linear layer with 512 input and output features + nn.ReLU(), # another relu activation function + nn.Linear(512,10) # a final linear layer with 512 input features and 10 output features + ) + + + def forward(self,x): # define the forward method + x = self.flatten(x) # flatten the input x + logits=self.linear_relu_stack(x) # pass x through the linear relu stack and get the logits + return logits # return the logits + + +class neuralnetwork: # define another class for the neural network object + def __init__(self): # define the constructor method + if torch.cuda.is_available(): # check if cuda is available + self.device=torch.device('cuda') # set the device to cuda + else: + self.device=torch.device('cpu') # otherwise set the device to cpu + self.model=NeuralNetwork().to(self.device) # create an instance of the NeuralNetwork class and move it to the device + self.loss_fn=nn.CrossEntropyLoss() # define the loss function as cross entropy loss + self.opt=[torch.optim.SGD(self.model.parameters(),lr=1e-3) for _ in range(7)] # define a list of three stochastic gradient descent optimizers with learning rate 1e-3 + + + def fp(self,x): # define a method for forward propagation + pred=self.model(x.to(self.device)) # get the predictions from the model by passing x to the device + return pred # return the predictions + + + def loss(self,output,labels): # define a method for calculating the loss + loss=self.loss_fn(output,labels.to(self.device)) # compute the loss by passing output and labels to the device + return loss # return the loss \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/pytorch/parallel/nn_device.py b/Note 7.0 documentation/DL/neural network/pytorch/parallel/nn_device.py new file mode 100644 index 00000000..5e73606a --- /dev/null +++ b/Note 7.0 documentation/DL/neural network/pytorch/parallel/nn_device.py @@ -0,0 +1,45 @@ +import torch # import the PyTorch library +from torch import nn # import the neural network module from PyTorch +from Note.nn.process.assign_device import assign_device_pytorch # import a custom function to assign device for PyTorch tensors + + +class NeuralNetwork(nn.Module): # define a class for the neural network model + def __init__(self): # define the constructor method + super().__init__() # call the parent class constructor + self.flatten = nn.Flatten() # define a layer to flatten the input tensor + self.linear_relu_stack=nn.Sequential( # define a stack of linear and relu layers + nn.Linear(28*28,512), # define a linear layer with input size 28*28 and output size 512 + nn.ReLU(), # define a relu activation function + nn.Linear(512,512), # define another linear layer with input and output size 512 + nn.ReLU(), # define another relu activation function + nn.Linear(512,10) # define the final linear layer with output size 10 + ) + + + def forward(self,x): # define the forward method for the model + x = self.flatten(x) # flatten the input tensor + logits=self.linear_relu_stack(x) # pass the flattened tensor through the linear relu stack + return logits # return the logits as the output + + +class neuralnetwork: # define another class for the neural network object + def __init__(self): # define the constructor method + if torch.cuda.is_available(): # check if cuda device is available + self.device=torch.device('cuda') # set the device to cuda + else: # otherwise + self.device=torch.device('cpu') # set the device to cpu + self.model=NeuralNetwork().to(self.device) # create an instance of the neural network model and move it to the device + self.loss_fn=nn.CrossEntropyLoss() # define the loss function as cross entropy loss + self.opt=[torch.optim.SGD(self.model.parameters(),lr=1e-3) for _ in range(7)] # define a list of optimizers as stochastic gradient descent with learning rate 1e-3 + + + def fp(self,x,p): # define a method for forward propagation + pred=self.model(x.to(assign_device_pytorch(p,'GPU'))) # assign the device according to the process index p + # get the predictions from the model by passing the input tensor to the device and then to the model + return pred # return the predictions + + + def loss(self,output,labels,p): # define a method for calculating loss + loss=self.loss_fn(output,labels.to(assign_device_pytorch(p,'GPU'))) # assign the device according to the process index p + # calculate the loss by passing the output and labels tensors to the device and then to the loss function + return loss # return the loss \ No newline at end of file From 27180f4c258f26c7cc8e5656fe012df0f107cd04 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 27 Jul 2023 13:45:16 +0800 Subject: [PATCH 110/337] Delete Note 7.0 documentation/DL/neural network/pytorch/process directory --- .../DL/neural network/pytorch/process/nn.py | 41 ----------------- .../pytorch/process/nn_device.py | 45 ------------------- 2 files changed, 86 deletions(-) delete mode 100644 Note 7.0 documentation/DL/neural network/pytorch/process/nn.py delete mode 100644 Note 7.0 documentation/DL/neural network/pytorch/process/nn_device.py diff --git a/Note 7.0 documentation/DL/neural network/pytorch/process/nn.py b/Note 7.0 documentation/DL/neural network/pytorch/process/nn.py deleted file mode 100644 index 56713cb0..00000000 --- a/Note 7.0 documentation/DL/neural network/pytorch/process/nn.py +++ /dev/null @@ -1,41 +0,0 @@ -import torch # import the PyTorch library -from torch import nn # import the neural network module from PyTorch - -class NeuralNetwork(nn.Module): # define a class for the neural network model - def __init__(self): # define the constructor method - super().__init__() # call the parent class constructor - self.flatten = nn.Flatten() # define a layer to flatten the input - self.linear_relu_stack=nn.Sequential( # define a stack of linear and relu layers - nn.Linear(28*28,512), # a linear layer with 28*28 input features and 512 output features - nn.ReLU(), # a relu activation function - nn.Linear(512,512), # another linear layer with 512 input and output features - nn.ReLU(), # another relu activation function - nn.Linear(512,10) # a final linear layer with 512 input features and 10 output features - ) - - - def forward(self,x): # define the forward method - x = self.flatten(x) # flatten the input x - logits=self.linear_relu_stack(x) # pass x through the linear relu stack and get the logits - return logits # return the logits - - -class neuralnetwork: # define another class for the neural network object - def __init__(self): # define the constructor method - if torch.cuda.is_available(): # check if cuda is available - self.device=torch.device('cuda') # set the device to cuda - else: - self.device=torch.device('cpu') # otherwise set the device to cpu - self.model=NeuralNetwork().to(self.device) # create an instance of the NeuralNetwork class and move it to the device - self.loss_fn=nn.CrossEntropyLoss() # define the loss function as cross entropy loss - self.opt=[torch.optim.SGD(self.model.parameters(),lr=1e-3) for _ in range(7)] # define a list of three stochastic gradient descent optimizers with learning rate 1e-3 - - - def fp(self,x): # define a method for forward propagation - pred=self.model(x.to(self.device)) # get the predictions from the model by passing x to the device - return pred # return the predictions - - - def loss(self,output,labels): # define a method for calculating the loss - loss=self.loss_fn(output,labels.to(self.device)) # compute the loss by passing output and labels to the device - return loss # return the loss diff --git a/Note 7.0 documentation/DL/neural network/pytorch/process/nn_device.py b/Note 7.0 documentation/DL/neural network/pytorch/process/nn_device.py deleted file mode 100644 index 9c807390..00000000 --- a/Note 7.0 documentation/DL/neural network/pytorch/process/nn_device.py +++ /dev/null @@ -1,45 +0,0 @@ -import torch # import the PyTorch library -from torch import nn # import the neural network module from PyTorch -from Note.nn.process.assign_device import assign_device_pytorch # import a custom function to assign device for PyTorch tensors - - -class NeuralNetwork(nn.Module): # define a class for the neural network model - def __init__(self): # define the constructor method - super().__init__() # call the parent class constructor - self.flatten = nn.Flatten() # define a layer to flatten the input tensor - self.linear_relu_stack=nn.Sequential( # define a stack of linear and relu layers - nn.Linear(28*28,512), # define a linear layer with input size 28*28 and output size 512 - nn.ReLU(), # define a relu activation function - nn.Linear(512,512), # define another linear layer with input and output size 512 - nn.ReLU(), # define another relu activation function - nn.Linear(512,10) # define the final linear layer with output size 10 - ) - - - def forward(self,x): # define the forward method for the model - x = self.flatten(x) # flatten the input tensor - logits=self.linear_relu_stack(x) # pass the flattened tensor through the linear relu stack - return logits # return the logits as the output - - -class neuralnetwork: # define another class for the neural network object - def __init__(self): # define the constructor method - if torch.cuda.is_available(): # check if cuda device is available - self.device=torch.device('cuda') # set the device to cuda - else: # otherwise - self.device=torch.device('cpu') # set the device to cpu - self.model=NeuralNetwork().to(self.device) # create an instance of the neural network model and move it to the device - self.loss_fn=nn.CrossEntropyLoss() # define the loss function as cross entropy loss - self.opt=[torch.optim.SGD(self.model.parameters(),lr=1e-3) for _ in range(7)] # define a list of optimizers as stochastic gradient descent with learning rate 1e-3 - - - def fp(self,x,p): # define a method for forward propagation - pred=self.model(x.to(assign_device_pytorch(p,'GPU'))) # assign the device according to the process index p - # get the predictions from the model by passing the input tensor to the device and then to the model - return pred # return the predictions - - - def loss(self,output,labels,p): # define a method for calculating loss - loss=self.loss_fn(output,labels.to(assign_device_pytorch(p,'GPU'))) # assign the device according to the process index p - # calculate the loss by passing the output and labels tensors to the device and then to the loss function - return loss # return the loss From 04421aa46b387f7de9876b1b37c1f2a03e06020e Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 27 Jul 2023 13:45:55 +0800 Subject: [PATCH 111/337] Delete nn.py --- .../DL/neural network/pytorch/nn.py | 53 ------------------- 1 file changed, 53 deletions(-) delete mode 100644 Note 7.0 documentation/DL/neural network/pytorch/nn.py diff --git a/Note 7.0 documentation/DL/neural network/pytorch/nn.py b/Note 7.0 documentation/DL/neural network/pytorch/nn.py deleted file mode 100644 index 1faec0fc..00000000 --- a/Note 7.0 documentation/DL/neural network/pytorch/nn.py +++ /dev/null @@ -1,53 +0,0 @@ -import torch -from torch import nn - - -class NeuralNetwork(nn.Module): - def __init__(self): - super().__init__() - self.flatten=nn.Flatten() - self.linear_relu_stack=nn.Sequential( - nn.Linear(28*28, 512), - nn.ReLU(), - nn.Linear(512, 512), - nn.ReLU(), - nn.Linear(512, 10) - ) - - - def forward(self,x): - x=self.flatten(x) - logits=self.linear_relu_stack(x) - return logits - - -class neuralnetwork: - def __init__(self): - if torch.cuda.is_available(): - self.device=torch.device('cuda') - else: - self.device=torch.device('cpu') - self.model=NeuralNetwork().to(self.device) - self.loss_fn=nn.CrossEntropyLoss() - self.opt=torch.optim.SGD(self.model.parameters(),lr=1e-3) #optimizer,kernel uses it to optimize. - - - def fp(self,x): #forward propagation function,kernel uses it for forward propagation. - pred=self.model(x.to(self.device)) - return pred - - - def loss(self,output,labels): #loss functino,kernel uses it to calculate loss. - loss=self.loss_fn(output,labels.to(self.device)) - return loss - - - def backward(self,loss): #backward function,kernel uses it for backpropagation. - self.optim.zero_grad() - loss.backward() - return - - - def opt(self): #opt function,kernel uses it to optimize. - self.optim.step() - return \ No newline at end of file From 1d66f344f46fba7a4ab16026247df1a3d8a56548 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 27 Jul 2023 13:46:59 +0800 Subject: [PATCH 112/337] Update nn_device.py --- .../DL/neural network/pytorch/parallel/nn_device.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Note 7.0 documentation/DL/neural network/pytorch/parallel/nn_device.py b/Note 7.0 documentation/DL/neural network/pytorch/parallel/nn_device.py index 5e73606a..bd0e8a7e 100644 --- a/Note 7.0 documentation/DL/neural network/pytorch/parallel/nn_device.py +++ b/Note 7.0 documentation/DL/neural network/pytorch/parallel/nn_device.py @@ -1,6 +1,6 @@ import torch # import the PyTorch library from torch import nn # import the neural network module from PyTorch -from Note.nn.process.assign_device import assign_device_pytorch # import a custom function to assign device for PyTorch tensors +from Note.nn.parallel.assign_device_pytorch import assign_device # import a custom function to assign device for PyTorch tensors class NeuralNetwork(nn.Module): # define a class for the neural network model @@ -34,12 +34,12 @@ def __init__(self): # define the constructor method def fp(self,x,p): # define a method for forward propagation - pred=self.model(x.to(assign_device_pytorch(p,'GPU'))) # assign the device according to the process index p + pred=self.model(x.to(assign_device(p,'GPU'))) # assign the device according to the process index p # get the predictions from the model by passing the input tensor to the device and then to the model return pred # return the predictions def loss(self,output,labels,p): # define a method for calculating loss - loss=self.loss_fn(output,labels.to(assign_device_pytorch(p,'GPU'))) # assign the device according to the process index p + loss=self.loss_fn(output,labels.to(assign_device(p,'GPU'))) # assign the device according to the process index p # calculate the loss by passing the output and labels tensors to the device and then to the loss function - return loss # return the loss \ No newline at end of file + return loss # return the loss From 81b398efd69e660e8dc5d10c16ad1b2dca02267f Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 27 Jul 2023 13:48:04 +0800 Subject: [PATCH 113/337] Add files via upload --- .../tensorflow/non-parallel/DDPG.py | 84 +++++++++++++++++++ .../tensorflow/non-parallel/DQN.py | 46 ++++++++++ .../tensorflow/non-parallel/DQN_pr.py | 59 +++++++++++++ .../neural network/tensorflow/parallel/DQN.py | 52 ++++++++++++ 4 files changed, 241 insertions(+) create mode 100644 Note 7.0 documentation/RL/neural network/tensorflow/non-parallel/DDPG.py create mode 100644 Note 7.0 documentation/RL/neural network/tensorflow/non-parallel/DQN.py create mode 100644 Note 7.0 documentation/RL/neural network/tensorflow/non-parallel/DQN_pr.py create mode 100644 Note 7.0 documentation/RL/neural network/tensorflow/parallel/DQN.py diff --git a/Note 7.0 documentation/RL/neural network/tensorflow/non-parallel/DDPG.py b/Note 7.0 documentation/RL/neural network/tensorflow/non-parallel/DDPG.py new file mode 100644 index 00000000..d3666ee1 --- /dev/null +++ b/Note 7.0 documentation/RL/neural network/tensorflow/non-parallel/DDPG.py @@ -0,0 +1,84 @@ +import tensorflow as tf # import TensorFlow library +import numpy as np # import NumPy library +import gym # import OpenAI Gym library + + +class actor: # define a class for the actor network + def __init__(self,state_dim,hidden_dim,action_dim,action_bound): # initialize the network with state dimension, hidden dimension, action dimension and action bound + self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) # create a weight matrix for the first layer + self.bias1=tf.Variable(tf.random.normal([hidden_dim])) # create a bias vector for the first layer + self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) # create a weight matrix for the second layer + self.bias2=tf.Variable(tf.random.normal([action_dim])) # create a bias vector for the second layer + self.action_bound=action_bound # store the action bound + self.param=[self.weight1,self.bias1,self.weight2,self.bias2] # store the network parameters in a list + + + def fp(self,x): # forward propagation function, kernel uses it for forward propagation + x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) # apply the first layer with ReLU activation + return tf.nn.tanh(tf.matmul(x,self.weight2)+self.bias2)*self.action_bound # apply the second layer with tanh activation and scale it by the action bound + + +class critic: # define a class for the critic network + def __init__(self,state_dim,hidden_dim,action_dim): # initialize the network with state dimension, hidden dimension and action dimension + self.weight1=tf.Variable(tf.random.normal([state_dim+action_dim,hidden_dim])) # create a weight matrix for the first layer + self.bias1=tf.Variable(tf.random.normal([hidden_dim])) # create a bias vector for the first layer + self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) # create a weight matrix for the second layer + self.bias2=tf.Variable(tf.random.normal([action_dim])) # create a bias vector for the second layer + self.param=[self.weight1,self.bias1,self.weight2,self.bias2] # store the network parameters in a list + + + def fp(self,x,a): # forward propagation function, kernel uses it for forward propagation + cat=tf.concat([x,a],axis=1) # concatenate the state and action vectors along the first axis + x=tf.nn.relu(tf.matmul(cat,self.weight1)+self.bias1) # apply the first layer with ReLU activation + return tf.matmul(x,self.weight2)+self.bias2 # apply the second layer and return the output + + +class DDPG: # define a class for the DDPG agent + def __init__(self,hidden_dim,sigma,gamma,tau,actor_lr,critic_lr): # initialize the agent with hidden dimension, noise scale, discount factor, soft update factor, actor learning rate and critic learning rate + self.genv=gym.make('Pendulum-v0') # create environment + state_dim=self.genv.observation_space.shape[0] # get the state dimension from the environment observation space + action_dim=self.genv.action_space.shape[0] # get the action dimension from the environment action space + action_bound=self.genv.action_space.high[0] # get the action bound from the environment action space high value (assume symmetric) + self.actor=actor(state_dim,hidden_dim,action_dim,action_bound) # create an actor network for the agent + self.critic=critic(state_dim,hidden_dim,action_dim) # create a critic network for the agent + self.target_actor=actor(state_dim,hidden_dim,action_dim,action_bound) # create a target actor network for the agent + self.target_critic=critic(state_dim,hidden_dim,action_dim) # create a target critic network for the agent + self.actor_param=self.actor.param # parameter list of actor network, kernel uses it list for backpropagation + self.critic_param=self.critic.param # parameter list of critic network, kernel uses it list for backpropagation + self.target_actor.param=self.actor_param.copy() # copy the parameters from actor network to target actor network + self.target_critic.param=self.critic_param.copy() # copy the parameters from critic network to target critic network + self.param=[self.actor_param,self.critic_param] # parameter list of both networks, kernel uses it list for backpropagation + self.sigma=sigma # noise scale + self.gamma=gamma # discount factor + self.tau=tau # soft update factor + self.opt=tf.keras.optimizers.Adam() # optimizer, kernel uses it to optimize. Here we use Adam optimizer + + + def noise(self): # noise function, kernel uses it to generate exploration noise + return np.random.normal(scale=self.sigma) # return a random sample from a normal distribution with zero mean and sigma scale + + + def env(self,a=None,initial=None): # environment function, kernel uses it to interact with the environment + if initial==True: # if initial is True, reset the environment and return the initial state + state=self.genv.reset(seed=0) + return state + else: # otherwise, take an action and return the next state, reward and done flag + next_state,reward,done,_=self.genv.step(a) + return next_state,reward,done + + + def loss(self,s,a,next_s,r,d): # loss function, kernel uses it to calculate loss + a=tf.expand_dims(a,axis=1) # expand the action vector to match the Q-value matrix shape + next_q_value=self.target_critic.fp(next_s,self.target_actor.fp(next_s)) # get the Q-value for the next state and action from the target critic network + q_target=r+self.gamma*next_q_value*(1-d) # calculate the target value using Bellman equation with discount factor gamma + actor_loss=-tf.reduce_mean(self.critic.fp(s,self.actor.fp(s))) # calculate the actor loss as the negative mean of the Q-value for the current state and action from the critic network + critic_loss=tf.reduce_mean((self.critic.fp(s,a)-q_target)**2) # calculate the critic loss as the mean squared error between Q-value and target value + return [actor_loss,critic_loss] # return a list of actor loss and critic loss + + + def update_param(self): # update function, kernel uses it to update parameter + for target_param,param in zip(self.target_actor.param,self.actor.param): # for each pair of parameters in target actor network and actor network + target_param=target_param*(1.0-self.tau)+param*self.tau # update the target parameter using soft update with factor tau + for target_param,param in zip(self.target_critic.param,self.critic.param): # for each pair of parameters in target critic network and critic network + target_param=target_param*(1.0-self.tau)+param*self.tau # update the target parameter using soft update with factor tau + return \ No newline at end of file diff --git a/Note 7.0 documentation/RL/neural network/tensorflow/non-parallel/DQN.py b/Note 7.0 documentation/RL/neural network/tensorflow/non-parallel/DQN.py new file mode 100644 index 00000000..21487674 --- /dev/null +++ b/Note 7.0 documentation/RL/neural network/tensorflow/non-parallel/DQN.py @@ -0,0 +1,46 @@ +import tensorflow as tf # import TensorFlow library +import gym # import OpenAI Gym library + + +class Qnet: # define a class for the Q-network + def __init__(self,state_dim,hidden_dim,action_dim): # initialize the network with state dimension, hidden dimension and action dimension + self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) # create a weight matrix for the first layer + self.bias1=tf.Variable(tf.random.normal([hidden_dim])) # create a bias vector for the first layer + self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) # create a weight matrix for the second layer + self.bias2=tf.Variable(tf.random.normal([action_dim])) # create a bias vector for the second layer + self.param=[self.weight1,self.bias1,self.weight2,self.bias2] # store the network parameters in a list + + def fp(self,x): # forward propagation function, kernel uses it for forward propagation + x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) # apply the first layer with ReLU activation + return tf.matmul(x,self.weight2)+self.bias2 # apply the second layer and return the output + + +class DQN: # define a class for the DQN agent + def __init__(self,state_dim,hidden_dim,action_dim): # initialize the agent with state dimension, hidden dimension and action dimension + self.nn=Qnet(state_dim,hidden_dim,action_dim) # create a Q-network for the agent + self.target_q_net=Qnet(state_dim,hidden_dim,action_dim) # create a target Q-network for the agent + self.param=self.nn.param # parameter list, kernel uses it list for backpropagation + self.opt=tf.keras.optimizers.Adam() # optimizer, kernel uses it to optimize + self.genv=gym.make('CartPole-v0') # create environment + + + def env(self,a=None,initial=None): # environment function, kernel uses it to interact with the environment + if initial==True: # if initial is True, reset the environment and return the initial state + state=self.genv.reset(seed=0) + return state + else: # otherwise, take an action and return the next state, reward and done flag + next_state,reward,done,_=self.genv.step(a) + return next_state,reward,done + + + def loss(self,s,a,next_s,r,d): # loss function, kernel uses it to calculate loss + a=tf.expand_dims(a,axis=1) # expand the action vector to match the Q-value matrix shape + q_value=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) # get the Q-value for the selected action + next_q_value=tf.reduce_max(self.target_q_net.fp(next_s),axis=1) # get the maximum Q-value for the next state from the target network + target=r+0.98*next_q_value*(1-d) # calculate the target value using Bellman equation with discount factor 0.98 + return tf.reduce_mean((q_value-target)**2) # return the mean squared error between Q-value and target value + + + def update_param(self): # update function, kernel uses it to update parameter + self.target_q_net.param=self.param.copy() # copy the parameters from the Q-network to the target network + return \ No newline at end of file diff --git a/Note 7.0 documentation/RL/neural network/tensorflow/non-parallel/DQN_pr.py b/Note 7.0 documentation/RL/neural network/tensorflow/non-parallel/DQN_pr.py new file mode 100644 index 00000000..77e8b8a9 --- /dev/null +++ b/Note 7.0 documentation/RL/neural network/tensorflow/non-parallel/DQN_pr.py @@ -0,0 +1,59 @@ +import tensorflow as tf +import gym +import Note.create.RL.rl.prioritized_replay as pr + + +class Qnet: + def __init__(self,state_dim,hidden_dim,action_dim): + self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) + self.bias1=tf.Variable(tf.random.normal([hidden_dim])) + self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) + self.bias2=tf.Variable(tf.random.normal([action_dim])) + self.param=[self.weight1,self.bias1,self.weight2,self.bias2] + + def fp(self,x): + x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) + return tf.matmul(x,self.weight2)+self.bias2 + + +class DQN: + def __init__(self,state_dim,hidden_dim,action_dim): + self.nn=Qnet(state_dim,hidden_dim,action_dim) + self.target_q_net=Qnet(state_dim,hidden_dim,action_dim) + self.param=self.nn.param + self.pr=pr.pr() + self.initial_TD=7 + self._epsilon=0.0007 + self.alpha=0.7 + self.opt=tf.keras.optimizers.Adam() + self.genv=gym.make('CartPole-v0') + + + def env(self,a=None,initial=None): + if initial==True: + state=self.genv.reset(seed=0) + return state + else: + next_state,reward,done,_=self.genv.step(a) + return next_state,reward,done + + + def data_func(self,state_pool,action_pool,next_state_pool,reward_pool,done_pool,batch): + s,a,next_s,r,d=self.pr.sample(state_pool,action_pool,next_state_pool,reward_pool,done_pool,self.epsilon,self.alpha,batch) + return s,a,next_s,r,d + + + def loss(self,s,a,next_s,r,d): + a=tf.expand_dims(a,axis=1) + q_value=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) + next_q_value=tf.reduce_max(self.target_q_net.fp(next_s),axis=1) + target=r+0.98*next_q_value*(1-d) + target=tf.expand_dims(target,axis=1) + TD=target-q_value + self.pr.update_TD(TD) + return tf.reduce_mean(TD**2) + + + def update_param(self): + self.target_q_net.param=self.param.copy() + return \ No newline at end of file diff --git a/Note 7.0 documentation/RL/neural network/tensorflow/parallel/DQN.py b/Note 7.0 documentation/RL/neural network/tensorflow/parallel/DQN.py new file mode 100644 index 00000000..d7756475 --- /dev/null +++ b/Note 7.0 documentation/RL/neural network/tensorflow/parallel/DQN.py @@ -0,0 +1,52 @@ +import tensorflow as tf # import TensorFlow library +import gym # import OpenAI Gym library +import Note.nn.parallel.optimizer as o # import Note's optimizer module + + +class Qnet: # define a class for the Q-network + def __init__(self,state_dim,hidden_dim,action_dim): # initialize the network with state dimension, hidden dimension and action dimension + self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) # create a weight matrix for the first layer + self.bias1=tf.Variable(tf.random.normal([hidden_dim])) # create a bias vector for the first layer + self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) # create a weight matrix for the second layer + self.bias2=tf.Variable(tf.random.normal([action_dim])) # create a bias vector for the second layer + self.param=[self.weight1,self.bias1,self.weight2,self.bias2] # store the network parameters in a list + + def fp(self,x): # forward propagation function, kernel uses it for forward propagation + x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) # apply the first layer with ReLU activation + return tf.matmul(x,self.weight2)+self.bias2 # apply the second layer and return the output + + +class DQN: # define a class for the DQN agent + def __init__(self,state_dim,hidden_dim,action_dim): # initialize the agent with state dimension, hidden dimension and action dimension + self.nn=Qnet(state_dim,hidden_dim,action_dim) # create a Q-network for the agent + self.target_q_net=Qnet(state_dim,hidden_dim,action_dim) # create a target Q-network for the agent + self.param=self.nn.param # parameter list, kernel uses it list for backpropagation + self.optimizer=o.Momentum(0.07,0.7) # optimizer, kernel uses it to optimize. Here we use a custom momentum optimizer with learning rate 0.07 and momentum 0.7 + self.genv=[gym.make('CartPole-v0') for _ in range(5)] # create a list of 5 environments + + + def env(self,a=None,p=None,initial=None): # environment function, kernel uses it to interact with the environment + if initial==True: # if initial is True, reset the environment with index p and return the initial state + state=self.genv[p].reset(seed=0) + return state + else: # otherwise, take an action and return the next state, reward and done flag from the environment with index p + next_state,reward,done,_=self.genv[p].step(a) + return next_state,reward,done + + + def loss(self,s,a,next_s,r,d): # loss function, kernel uses it to calculate loss + a=tf.expand_dims(a,axis=1) # expand the action vector to match the Q-value matrix shape + q_value=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) # get the Q-value for the selected action + next_q_value=tf.reduce_max(self.target_q_net.fp(next_s),axis=1) # get the maximum Q-value for the next state from the target network + target=r+0.98*next_q_value*(1-d) # calculate the target value using Bellman equation with discount factor 0.98 + return tf.reduce_mean((q_value-target)**2) # return the mean squared error between Q-value and target value + + + def update_param(self): # update function, kernel uses it to update parameter + self.target_q_net.param=self.param.copy() # copy the parameters from the Q-network to the target network + return + + + def opt(self,gradient): # optimization function, kernel uses it to optimize parameter + param=self.optimizer.opt(gradient,self.param) # apply the custom momentum optimizer to update the parameters using the gradient + return param \ No newline at end of file From 55c25399b928151f18671353946402a2684ead80 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 27 Jul 2023 13:48:24 +0800 Subject: [PATCH 114/337] Delete Note 7.0 documentation/RL/neural network/tensorflow/pool network directory --- .../tensorflow/pool network/DQN.py | 52 ------------------- 1 file changed, 52 deletions(-) delete mode 100644 Note 7.0 documentation/RL/neural network/tensorflow/pool network/DQN.py diff --git a/Note 7.0 documentation/RL/neural network/tensorflow/pool network/DQN.py b/Note 7.0 documentation/RL/neural network/tensorflow/pool network/DQN.py deleted file mode 100644 index 4bd4ba28..00000000 --- a/Note 7.0 documentation/RL/neural network/tensorflow/pool network/DQN.py +++ /dev/null @@ -1,52 +0,0 @@ -import tensorflow as tf # import TensorFlow library -import gym # import OpenAI Gym library -import Note.nn.process.optimizer as o # import Note's optimizer module - - -class Qnet: # define a class for the Q-network - def __init__(self,state_dim,hidden_dim,action_dim): # initialize the network with state dimension, hidden dimension and action dimension - self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) # create a weight matrix for the first layer - self.bias1=tf.Variable(tf.random.normal([hidden_dim])) # create a bias vector for the first layer - self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) # create a weight matrix for the second layer - self.bias2=tf.Variable(tf.random.normal([action_dim])) # create a bias vector for the second layer - self.param=[self.weight1,self.bias1,self.weight2,self.bias2] # store the network parameters in a list - - def fp(self,x): # forward propagation function, kernel uses it for forward propagation - x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) # apply the first layer with ReLU activation - return tf.matmul(x,self.weight2)+self.bias2 # apply the second layer and return the output - - -class DQN: # define a class for the DQN agent - def __init__(self,state_dim,hidden_dim,action_dim): # initialize the agent with state dimension, hidden dimension and action dimension - self.nn=Qnet(state_dim,hidden_dim,action_dim) # create a Q-network for the agent - self.target_q_net=Qnet(state_dim,hidden_dim,action_dim) # create a target Q-network for the agent - self.param=self.nn.param # parameter list, kernel uses it list for backpropagation - self.optimizer=o.Momentum(0.07,0.7) # optimizer, kernel uses it to optimize. Here we use a custom momentum optimizer with learning rate 0.07 and momentum 0.7 - self.genv=[gym.make('CartPole-v0') for _ in range(5)] # create a list of 5 environments - - - def env(self,a=None,p=None,initial=None): # environment function, kernel uses it to interact with the environment - if initial==True: # if initial is True, reset the environment with index p and return the initial state - state=self.genv[p].reset(seed=0) - return state - else: # otherwise, take an action and return the next state, reward and done flag from the environment with index p - next_state,reward,done,_=self.genv[p].step(a) - return next_state,reward,done - - - def loss(self,s,a,next_s,r,d): # loss function, kernel uses it to calculate loss - a=tf.expand_dims(a,axis=1) # expand the action vector to match the Q-value matrix shape - q_value=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) # get the Q-value for the selected action - next_q_value=tf.reduce_max(self.target_q_net.fp(next_s),axis=1) # get the maximum Q-value for the next state from the target network - target=r+0.98*next_q_value*(1-d) # calculate the target value using Bellman equation with discount factor 0.98 - return tf.reduce_mean((q_value-target)**2) # return the mean squared error between Q-value and target value - - - def update_param(self): # update function, kernel uses it to update parameter - self.target_q_net.param=self.param.copy() # copy the parameters from the Q-network to the target network - return - - - def opt(self,gradient): # optimization function, kernel uses it to optimize parameter - param=self.optimizer.opt(gradient,self.param) # apply the custom momentum optimizer to update the parameters using the gradient - return param From e95eb11adacb5b5778b05ca7fcafbf94c2cc93df Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 27 Jul 2023 13:48:35 +0800 Subject: [PATCH 115/337] Delete DDPG.py --- .../RL/neural network/tensorflow/DDPG.py | 84 ------------------- 1 file changed, 84 deletions(-) delete mode 100644 Note 7.0 documentation/RL/neural network/tensorflow/DDPG.py diff --git a/Note 7.0 documentation/RL/neural network/tensorflow/DDPG.py b/Note 7.0 documentation/RL/neural network/tensorflow/DDPG.py deleted file mode 100644 index 9f4b243a..00000000 --- a/Note 7.0 documentation/RL/neural network/tensorflow/DDPG.py +++ /dev/null @@ -1,84 +0,0 @@ -import tensorflow as tf # import TensorFlow library -import numpy as np # import NumPy library -import gym # import OpenAI Gym library - - -class actor: # define a class for the actor network - def __init__(self,state_dim,hidden_dim,action_dim,action_bound): # initialize the network with state dimension, hidden dimension, action dimension and action bound - self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) # create a weight matrix for the first layer - self.bias1=tf.Variable(tf.random.normal([hidden_dim])) # create a bias vector for the first layer - self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) # create a weight matrix for the second layer - self.bias2=tf.Variable(tf.random.normal([action_dim])) # create a bias vector for the second layer - self.action_bound=action_bound # store the action bound - self.param=[self.weight1,self.bias1,self.weight2,self.bias2] # store the network parameters in a list - - - def fp(self,x): # forward propagation function, kernel uses it for forward propagation - x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) # apply the first layer with ReLU activation - return tf.nn.tanh(tf.matmul(x,self.weight2)+self.bias2)*self.action_bound # apply the second layer with tanh activation and scale it by the action bound - - -class critic: # define a class for the critic network - def __init__(self,state_dim,hidden_dim,action_dim): # initialize the network with state dimension, hidden dimension and action dimension - self.weight1=tf.Variable(tf.random.normal([state_dim+action_dim,hidden_dim])) # create a weight matrix for the first layer - self.bias1=tf.Variable(tf.random.normal([hidden_dim])) # create a bias vector for the first layer - self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) # create a weight matrix for the second layer - self.bias2=tf.Variable(tf.random.normal([action_dim])) # create a bias vector for the second layer - self.param=[self.weight1,self.bias1,self.weight2,self.bias2] # store the network parameters in a list - - - def fp(self,x,a): # forward propagation function, kernel uses it for forward propagation - cat=tf.concat([x,a],axis=1) # concatenate the state and action vectors along the first axis - x=tf.nn.relu(tf.matmul(cat,self.weight1)+self.bias1) # apply the first layer with ReLU activation - return tf.matmul(x,self.weight2)+self.bias2 # apply the second layer and return the output - - -class DDPG: # define a class for the DDPG agent - def __init__(self,hidden_dim,sigma,gamma,tau,actor_lr,critic_lr): # initialize the agent with hidden dimension, noise scale, discount factor, soft update factor, actor learning rate and critic learning rate - self.genv=gym.make('Pendulum-v0') # create environment - state_dim=self.genv.observation_space.shape[0] # get the state dimension from the environment observation space - action_dim=self.genv.action_space.shape[0] # get the action dimension from the environment action space - action_bound=self.genv.action_space.high[0] # get the action bound from the environment action space high value (assume symmetric) - self.actor=actor(state_dim,hidden_dim,action_dim,action_bound) # create an actor network for the agent - self.critic=critic(state_dim,hidden_dim,action_dim) # create a critic network for the agent - self.target_actor=actor(state_dim,hidden_dim,action_dim,action_bound) # create a target actor network for the agent - self.target_critic=critic(state_dim,hidden_dim,action_dim) # create a target critic network for the agent - self.actor_param=self.actor.param # parameter list of actor network, kernel uses it list for backpropagation - self.critic_param=self.critic.param # parameter list of critic network, kernel uses it list for backpropagation - self.target_actor.param=self.actor_param.copy() # copy the parameters from actor network to target actor network - self.target_critic.param=self.critic_param.copy() # copy the parameters from critic network to target critic network - self.param=[self.actor_param,self.critic_param] # parameter list of both networks, kernel uses it list for backpropagation - self.sigma=sigma # noise scale - self.gamma=gamma # discount factor - self.tau=tau # soft update factor - self.opt=tf.keras.optimizers.Adam() # optimizer, kernel uses it to optimize. Here we use Adam optimizer - - - def noise(self): # noise function, kernel uses it to generate exploration noise - return np.random.normal(scale=self.sigma) # return a random sample from a normal distribution with zero mean and sigma scale - - - def env(self,a=None,initial=None): # environment function, kernel uses it to interact with the environment - if initial==True: # if initial is True, reset the environment and return the initial state - state=self.genv.reset(seed=0) - return state - else: # otherwise, take an action and return the next state, reward and done flag - next_state,reward,done,_=self.genv.step(a) - return next_state,reward,done - - - def loss(self,s,a,next_s,r,d): # loss function, kernel uses it to calculate loss - a=tf.expand_dims(a,axis=1) # expand the action vector to match the Q-value matrix shape - next_q_value=self.target_critic.fp(next_s,self.target_actor.fp(next_s)) # get the Q-value for the next state and action from the target critic network - q_target=r+self.gamma*next_q_value*(1-d) # calculate the target value using Bellman equation with discount factor gamma - actor_loss=-tf.reduce_mean(self.critic.fp(s,self.actor.fp(s))) # calculate the actor loss as the negative mean of the Q-value for the current state and action from the critic network - critic_loss=tf.reduce_mean((self.critic.fp(s,a)-q_target)**2) # calculate the critic loss as the mean squared error between Q-value and target value - return [actor_loss,critic_loss] # return a list of actor loss and critic loss - - - def update_param(self): # update function, kernel uses it to update parameter - for target_param,param in zip(self.target_actor.param,self.actor.param): # for each pair of parameters in target actor network and actor network - target_param=target_param*(1.0-self.tau)+param*self.tau # update the target parameter using soft update with factor tau - for target_param,param in zip(self.target_critic.param,self.critic.param): # for each pair of parameters in target critic network and critic network - target_param=target_param*(1.0-self.tau)+param*self.tau # update the target parameter using soft update with factor tau - return From cd191412fad174b2cd838fc96b3fe2adf8541f5c Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 27 Jul 2023 13:48:44 +0800 Subject: [PATCH 116/337] Delete DQN.py --- .../RL/neural network/tensorflow/DQN.py | 46 ------------------- 1 file changed, 46 deletions(-) delete mode 100644 Note 7.0 documentation/RL/neural network/tensorflow/DQN.py diff --git a/Note 7.0 documentation/RL/neural network/tensorflow/DQN.py b/Note 7.0 documentation/RL/neural network/tensorflow/DQN.py deleted file mode 100644 index 9b4adcde..00000000 --- a/Note 7.0 documentation/RL/neural network/tensorflow/DQN.py +++ /dev/null @@ -1,46 +0,0 @@ -import tensorflow as tf # import TensorFlow library -import gym # import OpenAI Gym library - - -class Qnet: # define a class for the Q-network - def __init__(self,state_dim,hidden_dim,action_dim): # initialize the network with state dimension, hidden dimension and action dimension - self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) # create a weight matrix for the first layer - self.bias1=tf.Variable(tf.random.normal([hidden_dim])) # create a bias vector for the first layer - self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) # create a weight matrix for the second layer - self.bias2=tf.Variable(tf.random.normal([action_dim])) # create a bias vector for the second layer - self.param=[self.weight1,self.bias1,self.weight2,self.bias2] # store the network parameters in a list - - def fp(self,x): # forward propagation function, kernel uses it for forward propagation - x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) # apply the first layer with ReLU activation - return tf.matmul(x,self.weight2)+self.bias2 # apply the second layer and return the output - - -class DQN: # define a class for the DQN agent - def __init__(self,state_dim,hidden_dim,action_dim): # initialize the agent with state dimension, hidden dimension and action dimension - self.nn=Qnet(state_dim,hidden_dim,action_dim) # create a Q-network for the agent - self.target_q_net=Qnet(state_dim,hidden_dim,action_dim) # create a target Q-network for the agent - self.param=self.nn.param # parameter list, kernel uses it list for backpropagation - self.opt=tf.keras.optimizers.Adam() # optimizer, kernel uses it to optimize - self.genv=gym.make('CartPole-v0') # create environment - - - def env(self,a=None,initial=None): # environment function, kernel uses it to interact with the environment - if initial==True: # if initial is True, reset the environment and return the initial state - state=self.genv.reset(seed=0) - return state - else: # otherwise, take an action and return the next state, reward and done flag - next_state,reward,done,_=self.genv.step(a) - return next_state,reward,done - - - def loss(self,s,a,next_s,r,d): # loss function, kernel uses it to calculate loss - a=tf.expand_dims(a,axis=1) # expand the action vector to match the Q-value matrix shape - q_value=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) # get the Q-value for the selected action - next_q_value=tf.reduce_max(self.target_q_net.fp(next_s),axis=1) # get the maximum Q-value for the next state from the target network - target=r+0.98*next_q_value*(1-d) # calculate the target value using Bellman equation with discount factor 0.98 - return tf.reduce_mean((q_value-target)**2) # return the mean squared error between Q-value and target value - - - def update_param(self): # update function, kernel uses it to update parameter - self.target_q_net.param=self.param.copy() # copy the parameters from the Q-network to the target network - return From 9b0f2207612e86370414010eb592a0b729456a07 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 27 Jul 2023 13:48:53 +0800 Subject: [PATCH 117/337] Delete DQN_pr.py --- .../RL/neural network/tensorflow/DQN_pr.py | 59 ------------------- 1 file changed, 59 deletions(-) delete mode 100644 Note 7.0 documentation/RL/neural network/tensorflow/DQN_pr.py diff --git a/Note 7.0 documentation/RL/neural network/tensorflow/DQN_pr.py b/Note 7.0 documentation/RL/neural network/tensorflow/DQN_pr.py deleted file mode 100644 index 77e8b8a9..00000000 --- a/Note 7.0 documentation/RL/neural network/tensorflow/DQN_pr.py +++ /dev/null @@ -1,59 +0,0 @@ -import tensorflow as tf -import gym -import Note.create.RL.rl.prioritized_replay as pr - - -class Qnet: - def __init__(self,state_dim,hidden_dim,action_dim): - self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) - self.bias1=tf.Variable(tf.random.normal([hidden_dim])) - self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) - self.bias2=tf.Variable(tf.random.normal([action_dim])) - self.param=[self.weight1,self.bias1,self.weight2,self.bias2] - - def fp(self,x): - x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) - return tf.matmul(x,self.weight2)+self.bias2 - - -class DQN: - def __init__(self,state_dim,hidden_dim,action_dim): - self.nn=Qnet(state_dim,hidden_dim,action_dim) - self.target_q_net=Qnet(state_dim,hidden_dim,action_dim) - self.param=self.nn.param - self.pr=pr.pr() - self.initial_TD=7 - self._epsilon=0.0007 - self.alpha=0.7 - self.opt=tf.keras.optimizers.Adam() - self.genv=gym.make('CartPole-v0') - - - def env(self,a=None,initial=None): - if initial==True: - state=self.genv.reset(seed=0) - return state - else: - next_state,reward,done,_=self.genv.step(a) - return next_state,reward,done - - - def data_func(self,state_pool,action_pool,next_state_pool,reward_pool,done_pool,batch): - s,a,next_s,r,d=self.pr.sample(state_pool,action_pool,next_state_pool,reward_pool,done_pool,self.epsilon,self.alpha,batch) - return s,a,next_s,r,d - - - def loss(self,s,a,next_s,r,d): - a=tf.expand_dims(a,axis=1) - q_value=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) - next_q_value=tf.reduce_max(self.target_q_net.fp(next_s),axis=1) - target=r+0.98*next_q_value*(1-d) - target=tf.expand_dims(target,axis=1) - TD=target-q_value - self.pr.update_TD(TD) - return tf.reduce_mean(TD**2) - - - def update_param(self): - self.target_q_net.param=self.param.copy() - return \ No newline at end of file From 1c52c1147a728ad96856d5103eb84db033427101 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 27 Jul 2023 13:49:44 +0800 Subject: [PATCH 118/337] Add files via upload --- .../pytorch/non-parallel/DDPG.py | 101 ++++++++++++++++++ .../pytorch/non-parallel/DQN.py | 64 +++++++++++ .../pytorch/non-parallel/DQN_pr.py | 76 +++++++++++++ .../pytorch/non-parallel/DoubleDQN.py | 65 +++++++++++ .../pytorch/non-parallel/DuelingDQN.py | 67 ++++++++++++ .../RL/neural network/pytorch/parallel/DQN.py | 65 +++++++++++ 6 files changed, 438 insertions(+) create mode 100644 Note 7.0 documentation/RL/neural network/pytorch/non-parallel/DDPG.py create mode 100644 Note 7.0 documentation/RL/neural network/pytorch/non-parallel/DQN.py create mode 100644 Note 7.0 documentation/RL/neural network/pytorch/non-parallel/DQN_pr.py create mode 100644 Note 7.0 documentation/RL/neural network/pytorch/non-parallel/DoubleDQN.py create mode 100644 Note 7.0 documentation/RL/neural network/pytorch/non-parallel/DuelingDQN.py create mode 100644 Note 7.0 documentation/RL/neural network/pytorch/parallel/DQN.py diff --git a/Note 7.0 documentation/RL/neural network/pytorch/non-parallel/DDPG.py b/Note 7.0 documentation/RL/neural network/pytorch/non-parallel/DDPG.py new file mode 100644 index 00000000..4b0abd94 --- /dev/null +++ b/Note 7.0 documentation/RL/neural network/pytorch/non-parallel/DDPG.py @@ -0,0 +1,101 @@ +import torch +import torch.nn.functional as F +import numpy as np +import gym + + +class actor(torch.nn.Module): + def __init__(self,state_dim,hidden_dim,action_dim,action_bound): + super(actor,self).__init__() + self.fc1=torch.nn.Linear(state_dim,hidden_dim) + self.fc2=torch.nn.Linear(hidden_dim,action_dim) + self.action_bound=action_bound + + + def forward(self,x): + x=F.relu(self.fc1(x)) + return torch.tanh(self.fc2(x))*self.action_bound + + +class critic(torch.nn.Module): + def __init__(self,state_dim,hidden_dim,action_dim): + super(critic,self).__init__() + self.fc1=torch.nn.Linear(state_dim+action_dim,hidden_dim) + self.fc2=torch.nn.Linear(hidden_dim,1) + + + def forward(self,x,a): + cat=torch.cat([x,a],dim=1) + x=F.relu(self.fc1(cat)) + return self.fc2(x) + + +class DDPG: + def __init__(self,hidden_dim,sigma,gamma,tau,actor_lr,critic_lr): + if torch.cuda.is_available(): + self.device=torch.device('cuda') + else: + self.device=torch.device('cpu') + self.genv=gym.make('Pendulum-v0') + state_dim=self.genv.observation_space.shape[0] + action_dim=self.genv.action_space.shape[0] + action_bound=self.genv.action_space.high[0] + self.actor=actor(state_dim,hidden_dim,action_dim,action_bound).to(self.device) + self.critic=critic(state_dim,hidden_dim,action_dim).to(self.device) + self.target_actor=actor(state_dim,hidden_dim,action_dim,action_bound).to(self.device) + self.target_critic=critic(state_dim,hidden_dim,action_dim).to(self.device) + self.target_actor.load_state_dict(self.actor.state_dict()) + self.target_critic.load_state_dict(self.critic.state_dict()) + self.sigma=sigma + self.gamma=gamma + self.tau=tau + self.actor_opt=torch.optim.Adam(self.actor.parameters(),lr=actor_lr) + self.critic_opt=torch.optim.Adam(self.critic.parameters(),lr=critic_lr) + + + def noise(self): + return np.random.normal(scale=self.sigma) + + + def env(self,a=None,initial=None): + if initial==True: + state=self.genv.reset(seed=0) + return state + else: + next_state,reward,done,_=self.genv.step(a) + return next_state,reward,done + + + def loss(self,s,a,next_s,r,d): + s=torch.tensor(s,dtype=torch.float).to(self.device) + a=torch.tensor(a,dtype=torch.float).view(-1,1).to(self.device) + next_s=torch.tensor(next_s,dtype=torch.float).to(self.device) + r=torch.tensor(r,dtype=torch.float).view(-1,1).to(self.device) + d=torch.tensor(d,dtype=torch.float).view(-1,1).to(self.device) + next_q_value=self.target_critic(next_s,self.target_actor(next_s)) + q_target=r+self.gamma*next_q_value*(1-d) + actor_loss=-torch.mean(self.critic(s,self.actor(s))) + critic_loss=F.mse_loss(self.critic(s,a),q_target) + return [actor_loss,critic_loss] + + + def backward(self,loss): + self.actor_opt.zero_grad() + loss[0].backward() + self.critic_opt.zero_grad() + loss[1].backward() + return + + + def opt(self): + self.actor_opt.step() + self.critic_opt.step() + return + + + def update_param(self): + for target_param,param in zip(self.target_actor.parameters(),self.actor.parameters()): + target_param.data.copy_(target_param.data*(1.0-self.tau)+param.data*self.tau) + for target_param,param in zip(self.target_critic.parameters(),self.critic.parameters()): + target_param.data.copy_(target_param.data*(1.0-self.tau)+param.data*self.tau) + return \ No newline at end of file diff --git a/Note 7.0 documentation/RL/neural network/pytorch/non-parallel/DQN.py b/Note 7.0 documentation/RL/neural network/pytorch/non-parallel/DQN.py new file mode 100644 index 00000000..a0e58689 --- /dev/null +++ b/Note 7.0 documentation/RL/neural network/pytorch/non-parallel/DQN.py @@ -0,0 +1,64 @@ +import torch +import gym +import torch.nn.functional as F + + +class Qnet(torch.nn.Module): + def __init__(self,state_dim,hidden_dim,action_dim): + super(Qnet,self).__init__() + self.fc1=torch.nn.Linear(state_dim,hidden_dim) + self.fc2=torch.nn.Linear(hidden_dim,action_dim) + + + def forward(self,x): + x=F.relu(self.fc1(x)) + return self.fc2(x) + + +class DQN: + def __init__(self,state_dim,hidden_dim,action_dim): + if torch.cuda.is_available(): + self.device=torch.device('cuda') + else: + self.device=torch.device('cpu') + self.nn=Qnet(state_dim,hidden_dim,action_dim).to(self.device) + self.target_q_net=Qnet(state_dim,hidden_dim,action_dim).to(self.device) + self.optimizer=torch.optim.Adam(self.nn.parameters(),lr=2e-3) #optimizer,kernel uses it to optimize. + self.genv=gym.make('CartPole-v0') #create environment + + + def env(self,a=None,initial=None): #environment function,kernel uses it to interact with the environment. + if initial==True: + state=self.genv.reset() + return state + else: + next_state,reward,done,_=self.genv.step(a) + return next_state,reward,done + + + def loss(self,s,a,next_s,r,d): #loss function,kernel uses it to calculate loss. + s=torch.tensor(s,dtype=torch.float).to(self.device) + a=torch.tensor(a,dtype=torch.int64).view(-1,1).to(self.device) + next_s=torch.tensor(next_s,dtype=torch.float).to(self.device) + r=torch.tensor(r,dtype=torch.float).view(-1,1).to(self.device) + d=torch.tensor(d,dtype=torch.float).view(-1,1).to(self.device) + q_value=self.nn(s).gather(1,a) + next_q_value=self.target_q_net(next_s).max(1)[0].view(-1,1) + target=r+0.98*next_q_value*(1-d) + return F.mse_loss(q_value,target) + + + def backward(self,loss): #backward function,kernel uses it for backpropagation. + self.optimizer.zero_grad() + loss.backward() + return + + + def opt(self): #opt function,kernel uses it to optimize. + self.optimizer.step() + return + + + def update_param(self): #update function,kernel uses it to update parameter. + self.target_q_net.load_state_dict(self.nn.state_dict()) + return \ No newline at end of file diff --git a/Note 7.0 documentation/RL/neural network/pytorch/non-parallel/DQN_pr.py b/Note 7.0 documentation/RL/neural network/pytorch/non-parallel/DQN_pr.py new file mode 100644 index 00000000..66c71847 --- /dev/null +++ b/Note 7.0 documentation/RL/neural network/pytorch/non-parallel/DQN_pr.py @@ -0,0 +1,76 @@ +import torch +import gym +import torch.nn.functional as F +import Note.create.RL.rl.prioritized_replay as pr + +#prioritized replay example +class Qnet(torch.nn.Module): + def __init__(self,state_dim,hidden_dim,action_dim): + super(Qnet,self).__init__() + self.fc1=torch.nn.Linear(state_dim,hidden_dim) + self.fc2=torch.nn.Linear(hidden_dim,action_dim) + + + def forward(self,x): + x=F.relu(self.fc1(x)) + return self.fc2(x) + + +class DQN: + def __init__(self,state_dim,hidden_dim,action_dim): + if torch.cuda.is_available(): + self.device=torch.device('cuda') + else: + self.device=torch.device('cpu') + self.nn=Qnet(state_dim,hidden_dim,action_dim).to(self.device) + self.target_q_net=Qnet(state_dim,hidden_dim,action_dim).to(self.device) + self.pr=pr.pr() + self.initial_TD=7 + self._epsilon=0.0007 + self.alpha=0.7 + self.optimizer=torch.optim.Adam(self.nn.parameters(),lr=2e-3) + self.genv=gym.make('CartPole-v0') + + + def env(self,a=None,initial=None): + if initial==True: + state=self.genv.reset(seed=0) + return state + else: + next_state,reward,done,_=self.genv.step(a) + return next_state,reward,done + + + def data_func(self,state_pool,action_pool,next_state_pool,reward_pool,done_pool,batch): + s,a,next_s,r,d=self.pr.sample(state_pool,action_pool,next_state_pool,reward_pool,done_pool,self.epsilon,self.alpha,batch) + return s,a,next_s,r,d + + + def loss(self,s,a,next_s,r,d): + s=torch.tensor(s,dtype=torch.float).to(self.device) + a=torch.tensor(a,dtype=torch.int64).view(-1,1).to(self.device) + next_s=torch.tensor(next_s,dtype=torch.float).to(self.device) + r=torch.tensor(r,dtype=torch.float).view(-1,1).to(self.device) + d=torch.tensor(d,dtype=torch.float).view(-1,1).to(self.device) + q_value=self.nn(s).gather(1,a) + next_q_value=self.target_q_net(next_s).max(1)[0].view(-1,1) + target=r+0.98*next_q_value*(1-d) + TD=target-q_value + self.pr.update_TD(TD) + return torch.mean(TD**2) + + + def backward(self,loss): + self.optimizer.zero_grad() + loss.backward() + return + + + def opt(self): + self.optimizer.step() + return + + + def update_param(self): + self.target_q_net.load_state_dict(self.nn.state_dict()) + return \ No newline at end of file diff --git a/Note 7.0 documentation/RL/neural network/pytorch/non-parallel/DoubleDQN.py b/Note 7.0 documentation/RL/neural network/pytorch/non-parallel/DoubleDQN.py new file mode 100644 index 00000000..31e606e8 --- /dev/null +++ b/Note 7.0 documentation/RL/neural network/pytorch/non-parallel/DoubleDQN.py @@ -0,0 +1,65 @@ +import torch +import gym +import torch.nn.functional as F + + +class Qnet(torch.nn.Module): + def __init__(self,state_dim,hidden_dim,action_dim): + super(Qnet,self).__init__() + self.fc1=torch.nn.Linear(state_dim,hidden_dim) + self.fc2=torch.nn.Linear(hidden_dim,action_dim) + + + def forward(self,x): + x=F.relu(self.fc1(x)) + return self.fc2(x) + + +class DoubleDQN: + def __init__(self,state_dim,hidden_dim,action_dim): + if torch.cuda.is_available(): + self.device=torch.device('cuda') + else: + self.device=torch.device('cpu') + self.nn=Qnet(state_dim,hidden_dim,action_dim).to(self.device) + self.target_q_net=Qnet(state_dim,hidden_dim,action_dim).to(self.device) + self.optimizer=torch.optim.Adam(self.nn.parameters(),lr=2e-3) + self.genv=gym.make('CartPole-v0') + + + def env(self,a=None,initial=None): + if initial==True: + state=self.genv.reset(seed=0) + return state + else: + next_state,reward,done,_=self.genv.step(a) + return next_state,reward,done + + + def loss(self,s,a,next_s,r,d): + s=torch.tensor(s,dtype=torch.float).to(self.device) + a=torch.tensor(a,dtype=torch.int64).view(-1,1).to(self.device) + next_s=torch.tensor(next_s,dtype=torch.float).to(self.device) + r=torch.tensor(r,dtype=torch.float).view(-1,1).to(self.device) + d=torch.tensor(d,dtype=torch.float).view(-1,1).to(self.device) + q_value=self.nn(s).gather(1,a) + max_action=self.nn(next_s).max(1)[1].view(-1,1) + next_q_value=self.target_q_net(next_s).gather(1,max_action) + target=r+0.98*next_q_value*(1-d) + return F.mse_loss(q_value,target) + + + def backward(self,loss): + self.optimizer.zero_grad() + loss.backward() + return + + + def opt(self): + self.optimizer.step() + return + + + def update_param(self): + self.target_q_net.load_state_dict(self.nn.state_dict()) + return \ No newline at end of file diff --git a/Note 7.0 documentation/RL/neural network/pytorch/non-parallel/DuelingDQN.py b/Note 7.0 documentation/RL/neural network/pytorch/non-parallel/DuelingDQN.py new file mode 100644 index 00000000..b81eeb37 --- /dev/null +++ b/Note 7.0 documentation/RL/neural network/pytorch/non-parallel/DuelingDQN.py @@ -0,0 +1,67 @@ +import torch +import gym +import torch.nn.functional as F + + +class VAnet(torch.nn.Module): + def __init__(self,state_dim,hidden_dim,action_dim): + super(VAnet,self).__init__() + self.fc1=torch.nn.Linear(state_dim,hidden_dim) + self.fc_A=torch.nn.Linear(hidden_dim,action_dim) + self.fc_V=torch.nn.Linear(hidden_dim,1) + + + def forward(self,x): + A=self.fc_A(F.relu(self.fc1(x))) + V=self.fc_V(F.relu(self.fc1(x))) + Q=V+A-A.mean(1).view(-1,1) + return Q + + +class DuelingDQN: + def __init__(self,state_dim,hidden_dim,action_dim): + if torch.cuda.is_available(): + self.device=torch.device('cuda') + else: + self.device=torch.device('cpu') + self.nn=VAnet(state_dim,hidden_dim,action_dim).to(self.device) + self.target_q_net=VAnet(state_dim,hidden_dim,action_dim).to(self.device) + self.optimizer=torch.optim.Adam(self.nn.parameters(),lr=2e-3) + self.genv=gym.make('CartPole-v0') + + + def env(self,a=None,initial=None): + if initial==True: + state=self.genv.reset(seed=0) + return state + else: + next_state,reward,done,_=self.genv.step(a) + return next_state,reward,done + + + def loss(self,s,a,next_s,r,d): + s=torch.tensor(s,dtype=torch.float).to(self.device) + a=torch.tensor(a,dtype=torch.int64).view(-1,1).to(self.device) + next_s=torch.tensor(next_s,dtype=torch.float).to(self.device) + r=torch.tensor(r,dtype=torch.float).view(-1,1).to(self.device) + d=torch.tensor(d,dtype=torch.float).view(-1,1).to(self.device) + q_value=self.nn(s).gather(1,a) + next_q_value=self.target_q_net(next_s).max(1)[0].view(-1,1) + target=r+0.98*next_q_value*(1-d) + return F.mse_loss(q_value,target) + + + def backward(self,loss): + self.optimizer.zero_grad() + loss.backward() + return + + + def opt(self): + self.optimizer.step() + return + + + def update_param(self): + self.target_q_net.load_state_dict(self.nn.state_dict()) + return \ No newline at end of file diff --git a/Note 7.0 documentation/RL/neural network/pytorch/parallel/DQN.py b/Note 7.0 documentation/RL/neural network/pytorch/parallel/DQN.py new file mode 100644 index 00000000..f4d8417d --- /dev/null +++ b/Note 7.0 documentation/RL/neural network/pytorch/parallel/DQN.py @@ -0,0 +1,65 @@ +import torch +import gym +import torch.nn.functional as F +from Note.nn.parallel.assign_device_pytorch import assign_device + + +class Qnet(torch.nn.Module): + def __init__(self,state_dim,hidden_dim,action_dim): + super(Qnet,self).__init__() + self.fc1=torch.nn.Linear(state_dim,hidden_dim) + self.fc2=torch.nn.Linear(hidden_dim,action_dim) + + + def forward(self,x): + x=F.relu(self.fc1(x)) + return self.fc2(x) + + +class DQN: + def __init__(self,state_dim,hidden_dim,action_dim): + if torch.cuda.is_available(): + self.device=torch.device('cuda') + else: + self.device=torch.device('cpu') + self.nn=Qnet(state_dim,hidden_dim,action_dim).to(self.device) + self.target_q_net=Qnet(state_dim,hidden_dim,action_dim).to(self.device) + self.optimizer=[torch.optim.Adam(self.nn.parameters(),lr=2e-3) for _ in range(5)] #optimizer,kernel uses it to optimize. + self.genv=[gym.make('CartPole-v0') for _ in range(5)] #create environment + + + def env(self,a=None,p=None,initial=None): #environment function,kernel uses it to interact with the environment. + if initial==True: + state=self.genv[p].reset() + return state + else: + next_state,reward,done,_=self.genv[p].step(a) + return next_state,reward,done + + + def loss(self,s,a,next_s,r,d,p): #loss function,kernel uses it to calculate loss. + s=torch.tensor(s,dtype=torch.float).to(assign_device(p,'GPU')) + a=torch.tensor(a,dtype=torch.int64).view(-1,1).to(assign_device(p,'GPU')) + next_s=torch.tensor(next_s,dtype=torch.float).to(assign_device(p,'GPU')) + r=torch.tensor(r,dtype=torch.float).view(-1,1).to(assign_device(p,'GPU')) + d=torch.tensor(d,dtype=torch.float).view(-1,1).to(assign_device(p,'GPU')) + q_value=self.nn(s).gather(1,a) + next_q_value=self.target_q_net(next_s).max(1)[0].view(-1,1) + target=r+0.98*next_q_value*(1-d) + return F.mse_loss(q_value,target) + + + def backward(self,loss,p): #backward function,kernel uses it for backpropagation. + self.optimizer[p].zero_grad(set_to_none=True) + loss.backward() + return + + + def opt(self,p): #opt function,kernel uses it to optimize. + self.optimizer[p].step() + return + + + def update_param(self): #update function,kernel uses it to update parameter. + self.target_q_net.load_state_dict(self.nn.state_dict()) + return \ No newline at end of file From 37c0d3fda1979e492db44cb9997c26233655eecb Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 27 Jul 2023 13:50:11 +0800 Subject: [PATCH 119/337] Delete Note 7.0 documentation/RL/neural network/pytorch/pool network directory --- .../pytorch/pool network/DQN.py | 65 ------------------- 1 file changed, 65 deletions(-) delete mode 100644 Note 7.0 documentation/RL/neural network/pytorch/pool network/DQN.py diff --git a/Note 7.0 documentation/RL/neural network/pytorch/pool network/DQN.py b/Note 7.0 documentation/RL/neural network/pytorch/pool network/DQN.py deleted file mode 100644 index 519cbef4..00000000 --- a/Note 7.0 documentation/RL/neural network/pytorch/pool network/DQN.py +++ /dev/null @@ -1,65 +0,0 @@ -import torch -import gym -import torch.nn.functional as F -from Note.nn.process.assign_device_pytorch import assign_device - - -class Qnet(torch.nn.Module): - def __init__(self,state_dim,hidden_dim,action_dim): - super(Qnet,self).__init__() - self.fc1=torch.nn.Linear(state_dim,hidden_dim) - self.fc2=torch.nn.Linear(hidden_dim,action_dim) - - - def forward(self,x): - x=F.relu(self.fc1(x)) - return self.fc2(x) - - -class DQN: - def __init__(self,state_dim,hidden_dim,action_dim): - if torch.cuda.is_available(): - self.device=torch.device('cuda') - else: - self.device=torch.device('cpu') - self.nn=Qnet(state_dim,hidden_dim,action_dim).to(self.device) - self.target_q_net=Qnet(state_dim,hidden_dim,action_dim).to(self.device) - self.optimizer=[torch.optim.Adam(self.nn.parameters(),lr=2e-3) for _ in range(5)] #optimizer,kernel uses it to optimize. - self.genv=[gym.make('CartPole-v0') for _ in range(5)] #create environment - - - def env(self,a=None,p=None,initial=None): #environment function,kernel uses it to interact with the environment. - if initial==True: - state=self.genv[p].reset() - return state - else: - next_state,reward,done,_=self.genv[p].step(a) - return next_state,reward,done - - - def loss(self,s,a,next_s,r,d,p): #loss function,kernel uses it to calculate loss. - s=torch.tensor(s,dtype=torch.float).to(assign_device(p,'GPU')) - a=torch.tensor(a,dtype=torch.int64).view(-1,1).to(assign_device(p,'GPU')) - next_s=torch.tensor(next_s,dtype=torch.float).to(assign_device(p,'GPU')) - r=torch.tensor(r,dtype=torch.float).view(-1,1).to(assign_device(p,'GPU')) - d=torch.tensor(d,dtype=torch.float).view(-1,1).to(assign_device(p,'GPU')) - q_value=self.nn(s).gather(1,a) - next_q_value=self.target_q_net(next_s).max(1)[0].view(-1,1) - target=r+0.98*next_q_value*(1-d) - return F.mse_loss(q_value,target) - - - def backward(self,loss,p): #backward function,kernel uses it for backpropagation. - self.optimizer[p].zero_grad(set_to_none=True) - loss.backward() - return - - - def opt(self,p): #opt function,kernel uses it to optimize. - self.optimizer[p].step() - return - - - def update_param(self): #update function,kernel uses it to update parameter. - self.target_q_net.load_state_dict(self.nn.state_dict()) - return From ab7cca4aa83db1c37f13d462e25246e678b92172 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 27 Jul 2023 13:50:23 +0800 Subject: [PATCH 120/337] Delete DDPG.py --- .../RL/neural network/pytorch/DDPG.py | 101 ------------------ 1 file changed, 101 deletions(-) delete mode 100644 Note 7.0 documentation/RL/neural network/pytorch/DDPG.py diff --git a/Note 7.0 documentation/RL/neural network/pytorch/DDPG.py b/Note 7.0 documentation/RL/neural network/pytorch/DDPG.py deleted file mode 100644 index 4b0abd94..00000000 --- a/Note 7.0 documentation/RL/neural network/pytorch/DDPG.py +++ /dev/null @@ -1,101 +0,0 @@ -import torch -import torch.nn.functional as F -import numpy as np -import gym - - -class actor(torch.nn.Module): - def __init__(self,state_dim,hidden_dim,action_dim,action_bound): - super(actor,self).__init__() - self.fc1=torch.nn.Linear(state_dim,hidden_dim) - self.fc2=torch.nn.Linear(hidden_dim,action_dim) - self.action_bound=action_bound - - - def forward(self,x): - x=F.relu(self.fc1(x)) - return torch.tanh(self.fc2(x))*self.action_bound - - -class critic(torch.nn.Module): - def __init__(self,state_dim,hidden_dim,action_dim): - super(critic,self).__init__() - self.fc1=torch.nn.Linear(state_dim+action_dim,hidden_dim) - self.fc2=torch.nn.Linear(hidden_dim,1) - - - def forward(self,x,a): - cat=torch.cat([x,a],dim=1) - x=F.relu(self.fc1(cat)) - return self.fc2(x) - - -class DDPG: - def __init__(self,hidden_dim,sigma,gamma,tau,actor_lr,critic_lr): - if torch.cuda.is_available(): - self.device=torch.device('cuda') - else: - self.device=torch.device('cpu') - self.genv=gym.make('Pendulum-v0') - state_dim=self.genv.observation_space.shape[0] - action_dim=self.genv.action_space.shape[0] - action_bound=self.genv.action_space.high[0] - self.actor=actor(state_dim,hidden_dim,action_dim,action_bound).to(self.device) - self.critic=critic(state_dim,hidden_dim,action_dim).to(self.device) - self.target_actor=actor(state_dim,hidden_dim,action_dim,action_bound).to(self.device) - self.target_critic=critic(state_dim,hidden_dim,action_dim).to(self.device) - self.target_actor.load_state_dict(self.actor.state_dict()) - self.target_critic.load_state_dict(self.critic.state_dict()) - self.sigma=sigma - self.gamma=gamma - self.tau=tau - self.actor_opt=torch.optim.Adam(self.actor.parameters(),lr=actor_lr) - self.critic_opt=torch.optim.Adam(self.critic.parameters(),lr=critic_lr) - - - def noise(self): - return np.random.normal(scale=self.sigma) - - - def env(self,a=None,initial=None): - if initial==True: - state=self.genv.reset(seed=0) - return state - else: - next_state,reward,done,_=self.genv.step(a) - return next_state,reward,done - - - def loss(self,s,a,next_s,r,d): - s=torch.tensor(s,dtype=torch.float).to(self.device) - a=torch.tensor(a,dtype=torch.float).view(-1,1).to(self.device) - next_s=torch.tensor(next_s,dtype=torch.float).to(self.device) - r=torch.tensor(r,dtype=torch.float).view(-1,1).to(self.device) - d=torch.tensor(d,dtype=torch.float).view(-1,1).to(self.device) - next_q_value=self.target_critic(next_s,self.target_actor(next_s)) - q_target=r+self.gamma*next_q_value*(1-d) - actor_loss=-torch.mean(self.critic(s,self.actor(s))) - critic_loss=F.mse_loss(self.critic(s,a),q_target) - return [actor_loss,critic_loss] - - - def backward(self,loss): - self.actor_opt.zero_grad() - loss[0].backward() - self.critic_opt.zero_grad() - loss[1].backward() - return - - - def opt(self): - self.actor_opt.step() - self.critic_opt.step() - return - - - def update_param(self): - for target_param,param in zip(self.target_actor.parameters(),self.actor.parameters()): - target_param.data.copy_(target_param.data*(1.0-self.tau)+param.data*self.tau) - for target_param,param in zip(self.target_critic.parameters(),self.critic.parameters()): - target_param.data.copy_(target_param.data*(1.0-self.tau)+param.data*self.tau) - return \ No newline at end of file From abc6b89c6b799ae11537c3faae5b69a7f1cdfad7 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 27 Jul 2023 13:50:34 +0800 Subject: [PATCH 121/337] Delete DQN.py --- .../RL/neural network/pytorch/DQN.py | 64 ------------------- 1 file changed, 64 deletions(-) delete mode 100644 Note 7.0 documentation/RL/neural network/pytorch/DQN.py diff --git a/Note 7.0 documentation/RL/neural network/pytorch/DQN.py b/Note 7.0 documentation/RL/neural network/pytorch/DQN.py deleted file mode 100644 index a0e58689..00000000 --- a/Note 7.0 documentation/RL/neural network/pytorch/DQN.py +++ /dev/null @@ -1,64 +0,0 @@ -import torch -import gym -import torch.nn.functional as F - - -class Qnet(torch.nn.Module): - def __init__(self,state_dim,hidden_dim,action_dim): - super(Qnet,self).__init__() - self.fc1=torch.nn.Linear(state_dim,hidden_dim) - self.fc2=torch.nn.Linear(hidden_dim,action_dim) - - - def forward(self,x): - x=F.relu(self.fc1(x)) - return self.fc2(x) - - -class DQN: - def __init__(self,state_dim,hidden_dim,action_dim): - if torch.cuda.is_available(): - self.device=torch.device('cuda') - else: - self.device=torch.device('cpu') - self.nn=Qnet(state_dim,hidden_dim,action_dim).to(self.device) - self.target_q_net=Qnet(state_dim,hidden_dim,action_dim).to(self.device) - self.optimizer=torch.optim.Adam(self.nn.parameters(),lr=2e-3) #optimizer,kernel uses it to optimize. - self.genv=gym.make('CartPole-v0') #create environment - - - def env(self,a=None,initial=None): #environment function,kernel uses it to interact with the environment. - if initial==True: - state=self.genv.reset() - return state - else: - next_state,reward,done,_=self.genv.step(a) - return next_state,reward,done - - - def loss(self,s,a,next_s,r,d): #loss function,kernel uses it to calculate loss. - s=torch.tensor(s,dtype=torch.float).to(self.device) - a=torch.tensor(a,dtype=torch.int64).view(-1,1).to(self.device) - next_s=torch.tensor(next_s,dtype=torch.float).to(self.device) - r=torch.tensor(r,dtype=torch.float).view(-1,1).to(self.device) - d=torch.tensor(d,dtype=torch.float).view(-1,1).to(self.device) - q_value=self.nn(s).gather(1,a) - next_q_value=self.target_q_net(next_s).max(1)[0].view(-1,1) - target=r+0.98*next_q_value*(1-d) - return F.mse_loss(q_value,target) - - - def backward(self,loss): #backward function,kernel uses it for backpropagation. - self.optimizer.zero_grad() - loss.backward() - return - - - def opt(self): #opt function,kernel uses it to optimize. - self.optimizer.step() - return - - - def update_param(self): #update function,kernel uses it to update parameter. - self.target_q_net.load_state_dict(self.nn.state_dict()) - return \ No newline at end of file From 12831a23ba38bb2bf465d5b036e321ec52a8f89b Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 27 Jul 2023 13:50:45 +0800 Subject: [PATCH 122/337] Delete DoubleDQN.py --- .../RL/neural network/pytorch/DoubleDQN.py | 65 ------------------- 1 file changed, 65 deletions(-) delete mode 100644 Note 7.0 documentation/RL/neural network/pytorch/DoubleDQN.py diff --git a/Note 7.0 documentation/RL/neural network/pytorch/DoubleDQN.py b/Note 7.0 documentation/RL/neural network/pytorch/DoubleDQN.py deleted file mode 100644 index 31e606e8..00000000 --- a/Note 7.0 documentation/RL/neural network/pytorch/DoubleDQN.py +++ /dev/null @@ -1,65 +0,0 @@ -import torch -import gym -import torch.nn.functional as F - - -class Qnet(torch.nn.Module): - def __init__(self,state_dim,hidden_dim,action_dim): - super(Qnet,self).__init__() - self.fc1=torch.nn.Linear(state_dim,hidden_dim) - self.fc2=torch.nn.Linear(hidden_dim,action_dim) - - - def forward(self,x): - x=F.relu(self.fc1(x)) - return self.fc2(x) - - -class DoubleDQN: - def __init__(self,state_dim,hidden_dim,action_dim): - if torch.cuda.is_available(): - self.device=torch.device('cuda') - else: - self.device=torch.device('cpu') - self.nn=Qnet(state_dim,hidden_dim,action_dim).to(self.device) - self.target_q_net=Qnet(state_dim,hidden_dim,action_dim).to(self.device) - self.optimizer=torch.optim.Adam(self.nn.parameters(),lr=2e-3) - self.genv=gym.make('CartPole-v0') - - - def env(self,a=None,initial=None): - if initial==True: - state=self.genv.reset(seed=0) - return state - else: - next_state,reward,done,_=self.genv.step(a) - return next_state,reward,done - - - def loss(self,s,a,next_s,r,d): - s=torch.tensor(s,dtype=torch.float).to(self.device) - a=torch.tensor(a,dtype=torch.int64).view(-1,1).to(self.device) - next_s=torch.tensor(next_s,dtype=torch.float).to(self.device) - r=torch.tensor(r,dtype=torch.float).view(-1,1).to(self.device) - d=torch.tensor(d,dtype=torch.float).view(-1,1).to(self.device) - q_value=self.nn(s).gather(1,a) - max_action=self.nn(next_s).max(1)[1].view(-1,1) - next_q_value=self.target_q_net(next_s).gather(1,max_action) - target=r+0.98*next_q_value*(1-d) - return F.mse_loss(q_value,target) - - - def backward(self,loss): - self.optimizer.zero_grad() - loss.backward() - return - - - def opt(self): - self.optimizer.step() - return - - - def update_param(self): - self.target_q_net.load_state_dict(self.nn.state_dict()) - return \ No newline at end of file From 9a2e3e73729f4efccfdb97cf2ef4f1b0f7949480 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 27 Jul 2023 13:50:54 +0800 Subject: [PATCH 123/337] Delete DQN_pr.py --- .../RL/neural network/pytorch/DQN_pr.py | 76 ------------------- 1 file changed, 76 deletions(-) delete mode 100644 Note 7.0 documentation/RL/neural network/pytorch/DQN_pr.py diff --git a/Note 7.0 documentation/RL/neural network/pytorch/DQN_pr.py b/Note 7.0 documentation/RL/neural network/pytorch/DQN_pr.py deleted file mode 100644 index 66c71847..00000000 --- a/Note 7.0 documentation/RL/neural network/pytorch/DQN_pr.py +++ /dev/null @@ -1,76 +0,0 @@ -import torch -import gym -import torch.nn.functional as F -import Note.create.RL.rl.prioritized_replay as pr - -#prioritized replay example -class Qnet(torch.nn.Module): - def __init__(self,state_dim,hidden_dim,action_dim): - super(Qnet,self).__init__() - self.fc1=torch.nn.Linear(state_dim,hidden_dim) - self.fc2=torch.nn.Linear(hidden_dim,action_dim) - - - def forward(self,x): - x=F.relu(self.fc1(x)) - return self.fc2(x) - - -class DQN: - def __init__(self,state_dim,hidden_dim,action_dim): - if torch.cuda.is_available(): - self.device=torch.device('cuda') - else: - self.device=torch.device('cpu') - self.nn=Qnet(state_dim,hidden_dim,action_dim).to(self.device) - self.target_q_net=Qnet(state_dim,hidden_dim,action_dim).to(self.device) - self.pr=pr.pr() - self.initial_TD=7 - self._epsilon=0.0007 - self.alpha=0.7 - self.optimizer=torch.optim.Adam(self.nn.parameters(),lr=2e-3) - self.genv=gym.make('CartPole-v0') - - - def env(self,a=None,initial=None): - if initial==True: - state=self.genv.reset(seed=0) - return state - else: - next_state,reward,done,_=self.genv.step(a) - return next_state,reward,done - - - def data_func(self,state_pool,action_pool,next_state_pool,reward_pool,done_pool,batch): - s,a,next_s,r,d=self.pr.sample(state_pool,action_pool,next_state_pool,reward_pool,done_pool,self.epsilon,self.alpha,batch) - return s,a,next_s,r,d - - - def loss(self,s,a,next_s,r,d): - s=torch.tensor(s,dtype=torch.float).to(self.device) - a=torch.tensor(a,dtype=torch.int64).view(-1,1).to(self.device) - next_s=torch.tensor(next_s,dtype=torch.float).to(self.device) - r=torch.tensor(r,dtype=torch.float).view(-1,1).to(self.device) - d=torch.tensor(d,dtype=torch.float).view(-1,1).to(self.device) - q_value=self.nn(s).gather(1,a) - next_q_value=self.target_q_net(next_s).max(1)[0].view(-1,1) - target=r+0.98*next_q_value*(1-d) - TD=target-q_value - self.pr.update_TD(TD) - return torch.mean(TD**2) - - - def backward(self,loss): - self.optimizer.zero_grad() - loss.backward() - return - - - def opt(self): - self.optimizer.step() - return - - - def update_param(self): - self.target_q_net.load_state_dict(self.nn.state_dict()) - return \ No newline at end of file From 513d0a7f961f995a6c0022c7765cf9f931f19477 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 27 Jul 2023 13:51:03 +0800 Subject: [PATCH 124/337] Delete DuelingDQN.py --- .../RL/neural network/pytorch/DuelingDQN.py | 67 ------------------- 1 file changed, 67 deletions(-) delete mode 100644 Note 7.0 documentation/RL/neural network/pytorch/DuelingDQN.py diff --git a/Note 7.0 documentation/RL/neural network/pytorch/DuelingDQN.py b/Note 7.0 documentation/RL/neural network/pytorch/DuelingDQN.py deleted file mode 100644 index b81eeb37..00000000 --- a/Note 7.0 documentation/RL/neural network/pytorch/DuelingDQN.py +++ /dev/null @@ -1,67 +0,0 @@ -import torch -import gym -import torch.nn.functional as F - - -class VAnet(torch.nn.Module): - def __init__(self,state_dim,hidden_dim,action_dim): - super(VAnet,self).__init__() - self.fc1=torch.nn.Linear(state_dim,hidden_dim) - self.fc_A=torch.nn.Linear(hidden_dim,action_dim) - self.fc_V=torch.nn.Linear(hidden_dim,1) - - - def forward(self,x): - A=self.fc_A(F.relu(self.fc1(x))) - V=self.fc_V(F.relu(self.fc1(x))) - Q=V+A-A.mean(1).view(-1,1) - return Q - - -class DuelingDQN: - def __init__(self,state_dim,hidden_dim,action_dim): - if torch.cuda.is_available(): - self.device=torch.device('cuda') - else: - self.device=torch.device('cpu') - self.nn=VAnet(state_dim,hidden_dim,action_dim).to(self.device) - self.target_q_net=VAnet(state_dim,hidden_dim,action_dim).to(self.device) - self.optimizer=torch.optim.Adam(self.nn.parameters(),lr=2e-3) - self.genv=gym.make('CartPole-v0') - - - def env(self,a=None,initial=None): - if initial==True: - state=self.genv.reset(seed=0) - return state - else: - next_state,reward,done,_=self.genv.step(a) - return next_state,reward,done - - - def loss(self,s,a,next_s,r,d): - s=torch.tensor(s,dtype=torch.float).to(self.device) - a=torch.tensor(a,dtype=torch.int64).view(-1,1).to(self.device) - next_s=torch.tensor(next_s,dtype=torch.float).to(self.device) - r=torch.tensor(r,dtype=torch.float).view(-1,1).to(self.device) - d=torch.tensor(d,dtype=torch.float).view(-1,1).to(self.device) - q_value=self.nn(s).gather(1,a) - next_q_value=self.target_q_net(next_s).max(1)[0].view(-1,1) - target=r+0.98*next_q_value*(1-d) - return F.mse_loss(q_value,target) - - - def backward(self,loss): - self.optimizer.zero_grad() - loss.backward() - return - - - def opt(self): - self.optimizer.step() - return - - - def update_param(self): - self.target_q_net.load_state_dict(self.nn.state_dict()) - return \ No newline at end of file From 3abc066250b1e55125a79f5e0a2e48182058cd0a Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 27 Jul 2023 14:31:34 +0800 Subject: [PATCH 125/337] Update README.md --- README.md | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 94676fde..e4a652d9 100644 --- a/README.md +++ b/README.md @@ -105,7 +105,7 @@ kernel.train(32,5,32) #train the network with batch size 32, epoch 5 and ### PO2: ```python -import Note.DL.process.kernel as k #import kernel module +import Note.DL.parallel.kernel as k #import kernel module import tensorflow as tf #import tensorflow library import nn as n #import neural network module from multiprocessing import Process,Lock,Manager #import multiprocessing tools @@ -132,7 +132,7 @@ kernel.update_nn_param() #update the network parameters after traini kernel.test(x_train,y_train,32) #test the network performance on the train set with batch size 32 ``` ```python -import Note.DL.process.kernel as k #import kernel module +import Note.DL.parallel.kernel as k #import kernel module import tensorflow as tf #import tensorflow library import nn as n #import neural network module from multiprocessing import Process,Lock,Manager #import multiprocessing tools @@ -161,7 +161,7 @@ kernel.test(x_train,y_train,32) #test the network performance on the train ### PO3： ```python -import Note.DL.process.kernel as k #import kernel module +import Note.DL.parallel.kernel as k #import kernel module import tensorflow as tf #import tensorflow library import nn as n #import neural network module from multiprocessing import Process,Lock,Manager #import multiprocessing tools @@ -186,7 +186,7 @@ kernel.update_nn_param() #update the network parameters after traini kernel.test(x_train,y_train,32) #test the network performance on the train set with batch size 32 ``` ```python -import Note.DL.process.kernel as k #import kernel module +import Note.DL.parallel.kernel as k #import kernel module import tensorflow as tf #import tensorflow library import nn as n #import neural network module from multiprocessing import Process,Lock,Manager #import multiprocessing tools @@ -213,7 +213,7 @@ kernel.test(x_train,y_train,32) #test the network performance on the train ### Save and restore: ```python -import Note.DL.process.kernel as k #import kernel module +import Note.DL.parallel.kernel as k #import kernel module import tensorflow as tf #import tensorflow library import nn as n #import neural network module from multiprocessing import Process,Lock,Manager #import multiprocessing tools @@ -238,7 +238,7 @@ kernel.test(x_train,y_train,32) #test the network performance on the train kernel.save() #save the neural network to a file ``` ```python -import Note.DL.process.kernel as k #import kernel module +import Note.DL.parallel.kernel as k #import kernel module import tensorflow as tf #import tensorflow library import nn as n #import neural network module from multiprocessing import Process,Lock,Manager #import multiprocessing tools @@ -261,7 +261,7 @@ for p in range(7): #loop over the processes ### Parallel test: ```python -import Note.DL.process.kernel as k #import kernel module +import Note.DL.parallel.kernel as k #import kernel module import tensorflow as tf #import tensorflow library import nn as n #import neural network module from multiprocessing import Process,Manager #import multiprocessing tools @@ -286,7 +286,7 @@ for p in range(7): #loop over the processes ### Stop training and saving when condition is met: ```python -import Note.DL.process.kernel as k #import kernel module +import Note.DL.parallel.kernel as k #import kernel module import tensorflow as tf #import tensorflow library import nn as n #import neural network module from multiprocessing import Process,Lock,Manager #import multiprocessing tools @@ -318,11 +318,11 @@ for p in range(7): #loop over the processes **You can get neural network example from the link below, and then you can import neural network and train with kernel, example code are below.** -https://github.com/NoteDancing/Note-documentation/blob/Note-7.0/Note%207.0%20documentation/RL/neural%20network/tensorflow/pool%20network/DQN.py +https://github.com/NoteDancing/Note-documentation/blob/Note-7.0/Note%207.0%20documentation/RL/neural%20network/tensorflow/parallel/DQN.py ### PO2: ```python -import Note.RL.kernel as k #import kernel module +import Note.RL.parallel.kernel as k #import kernel module import DQN as d #import deep Q-network module from multiprocessing import Process,Lock,Manager #import multiprocessing tools dqn=d.DQN(4,128,2) #create neural network object with 4 inputs, 128 hidden units and 2 outputs @@ -341,7 +341,7 @@ for p in range(5): #loop over the processes ### PO3: ```python -import Note.RL.kernel as k #import kernel module +import Note.RL.parallel.kernel as k #import kernel module import DQN as d #import deep Q-network module from multiprocessing import Process,Lock,Manager #import multiprocessing tools dqn=d.DQN(4,128,2) #create neural network object with 4 inputs, 128 hidden units and 2 outputs @@ -357,7 +357,7 @@ for p in range(5): #loop over the processes Process(target=kernel.train,args=(p,100,lock,pool_lock)).start() #start each process with the train function and pass the process id, the number of episodes, the locks and the pool locks as arguments ``` ```python -import Note.RL.kernel as k #import kernel module +import Note.RL.parallel.kernel as k #import kernel module import DQN as d #import deep Q-network module from multiprocessing import Process,Lock,Manager #import multiprocessing tools dqn=d.DQN(4,128,2) #create neural network object with 4 inputs, 128 hidden units and 2 outputs @@ -377,7 +377,7 @@ for p in range(5): #loop over the processes # Parallel test: **You can get neural network example from the link below, and then you can import neural network and train with kernel, example code are below.** -https://github.com/NoteDancing/Note-documentation/blob/Note-7.0/Note%207.0%20documentation/DL/neural%20network/tensorflow/process/nn.py +https://github.com/NoteDancing/Note-documentation/blob/Note-7.0/Note%207.0%20documentation/DL/neural%20network/tensorflow/parallel/nn.py ```python import tensorflow as tf #import tensorflow library @@ -400,7 +400,7 @@ loss=test.loss_acc() #calculate the loss and accuracy of the test # Online training: **You can get neural network example from the link below, and then you can import neural network and train with kernel, example code are below.** -https://github.com/NoteDancing/Note-documentation/blob/Note-7.0/Note%207.0%20documentation/DL/neural%20network/tensorflow/nn_ol.py +https://github.com/NoteDancing/Note-documentation/blob/Note-7.0/Note%207.0%20documentation/DL/neural%20network/tensorflow/non-parallel/nn_ol.py **example:** ```python From 723fc7ba10b633778f7e57012c5cdc370c82ca3f Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 27 Jul 2023 14:33:57 +0800 Subject: [PATCH 126/337] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e4a652d9..06f22c36 100644 --- a/README.md +++ b/README.md @@ -80,7 +80,7 @@ kernel.test(x_test,y_test,32)#test the network performance on the test set with ## Parallel test: **You can get neural network example from the link below, and then you can import neural network and train with kernel, example code are below.** -https://github.com/NoteDancing/Note-documentation/blob/Note-7.0/Note%207.0%20documentation/DL/neural%20network/tensorflow/process/nn.py +https://github.com/NoteDancing/Note-documentation/blob/Note-7.0/Note%207.0%20documentation/DL/neural%20network/tensorflow/parallel/nn.py ```python import Note.DL.kernel as k #import kernel module From 4500d97ce73280be44324653be0badd999005aef Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 27 Jul 2023 20:18:37 +0800 Subject: [PATCH 127/337] Update README.md --- README.md | 56 ++++++++++++++++++++++++++++--------------------------- 1 file changed, 29 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 06f22c36..da135200 100644 --- a/README.md +++ b/README.md @@ -103,7 +103,7 @@ kernel.train(32,5,32) #train the network with batch size 32, epoch 5 and ## DL: -### PO2: +### PO1: ```python import Note.DL.parallel.kernel as k #import kernel module import tensorflow as tf #import tensorflow library @@ -112,7 +112,6 @@ from multiprocessing import Process,Lock,Manager #import multiprocessing tools mnist=tf.keras.datasets.mnist #load mnist dataset (x_train,y_train),(x_test,y_test)=mnist.load_data() #split data into train and test sets x_train,x_test =x_train/255.0,x_test/255.0 #normalize data -x_train=x_train.reshape([60000,784]) #reshape data to fit the network input nn=n.nn() #create neural network object nn.build() #build the network structure kernel=k.kernel(nn) #create kernel object with the network @@ -120,17 +119,18 @@ kernel.process=7 #set the number of processes to train kernel.data_segment_flag=True #set the flag to segment data for each process kernel.epoch=6 #set the number of epochs to train kernel.batch=32 #set the batch size -kernel.PO=2 #use PO2 algorithm for parallel optimization +kernel.PO=1 #use PO1 algorithm for parallel optimization kernel.data(x_train,y_train) #input train data to the kernel manager=Manager() #create manager object to share data among processes kernel.init(manager) #initialize shared data with the manager lock=[Lock(),Lock()] #create two locks for synchronization -g_lock=Lock() #create a global lock for gradient computing for p in range(7): #loop over the processes - Process(target=kernel.train,args=(p,lock,g_lock)).start() #start each process with the train function and pass the process id and locks as arguments + Process(target=kernel.train,args=(p,lock)).start() #start each process with the train function and pass the process id and locks as arguments kernel.update_nn_param() #update the network parameters after training kernel.test(x_train,y_train,32) #test the network performance on the train set with batch size 32 ``` + +### PO2: ```python import Note.DL.parallel.kernel as k #import kernel module import tensorflow as tf #import tensorflow library @@ -152,14 +152,12 @@ kernel.data(x_train,y_train) #input train data to the kernel manager=Manager() #create manager object to share data among processes kernel.init(manager) #initialize shared data with the manager lock=[Lock(),Lock()] #create two locks for synchronization -g_lock=[Lock(),Lock()] #create a list of global locks for gradient computing +g_lock=Lock() #create a global lock for gradient computing for p in range(7): #loop over the processes - Process(target=kernel.train,args=(p,lock,g_lock)).start() #start each process with the train function and pass the process id, the locks and the global locks as arguments + Process(target=kernel.train,args=(p,lock,g_lock)).start() #start each process with the train function and pass the process id and locks as arguments kernel.update_nn_param() #update the network parameters after training kernel.test(x_train,y_train,32) #test the network performance on the train set with batch size 32 ``` - -### PO3： ```python import Note.DL.parallel.kernel as k #import kernel module import tensorflow as tf #import tensorflow library @@ -168,6 +166,7 @@ from multiprocessing import Process,Lock,Manager #import multiprocessing tools mnist=tf.keras.datasets.mnist #load mnist dataset (x_train,y_train),(x_test,y_test)=mnist.load_data() #split data into train and test sets x_train,x_test =x_train/255.0,x_test/255.0 #normalize data +x_train=x_train.reshape([60000,784]) #reshape data to fit the network input nn=n.nn() #create neural network object nn.build() #build the network structure kernel=k.kernel(nn) #create kernel object with the network @@ -175,16 +174,19 @@ kernel.process=7 #set the number of processes to train kernel.data_segment_flag=True #set the flag to segment data for each process kernel.epoch=6 #set the number of epochs to train kernel.batch=32 #set the batch size -kernel.PO=3 #use PO3 algorithm for parallel optimization +kernel.PO=2 #use PO2 algorithm for parallel optimization kernel.data(x_train,y_train) #input train data to the kernel manager=Manager() #create manager object to share data among processes kernel.init(manager) #initialize shared data with the manager -lock=Lock() #create a lock for synchronization +lock=[Lock(),Lock()] #create two locks for synchronization +g_lock=[Lock(),Lock()] #create a list of global locks for gradient computing for p in range(7): #loop over the processes - Process(target=kernel.train,args=(p,lock)).start() #start each process with the train function and pass the process id and the lock as arguments + Process(target=kernel.train,args=(p,lock,g_lock)).start() #start each process with the train function and pass the process id, the locks and the global locks as arguments kernel.update_nn_param() #update the network parameters after training kernel.test(x_train,y_train,32) #test the network performance on the train set with batch size 32 ``` + +### PO3： ```python import Note.DL.parallel.kernel as k #import kernel module import tensorflow as tf #import tensorflow library @@ -204,11 +206,11 @@ kernel.PO=3 #use PO3 algorithm for parallel optimizatio kernel.data(x_train,y_train) #input train data to the kernel manager=Manager() #create manager object to share data among processes kernel.init(manager) #initialize shared data with the manager +lock=Lock() #create a lock for synchronization for p in range(7): #loop over the processes - Process(target=kernel.train,args=(p,)).start() #start each process with the train function and pass the process id as argument + Process(target=kernel.train,args=(p,lock)).start() #start each process with the train function and pass the process id and the lock as arguments kernel.update_nn_param() #update the network parameters after training kernel.test(x_train,y_train,32) #test the network performance on the train set with batch size 32 - ``` ### Save and restore: @@ -303,14 +305,12 @@ kernel.process=7 #set the number of processes to train kernel.data_segment_flag=True #set the flag to segment data for each process kernel.epoch=6 #set the number of epochs to train kernel.batch=32 #set the batch size -kernel.PO=2 #use PO2 algorithm for parallel optimization +kernel.PO=3 #use PO3 algorithm for parallel optimization kernel.data(x_train,y_train) #input train data to the kernel manager=Manager() #create manager object to share data among processes kernel.init(manager) #initialize shared data with the manager -lock=[Lock(),Lock()] #create two locks for synchronization -g_lock=Lock() #create a global lock for gradient update for p in range(7): #loop over the processes - Process(target=kernel.train,args=(p,lock,g_lock)).start() #start each process with the train function and pass the process id and locks as arguments + Process(target=kernel.train,args=(p,)).start() #start each process with the train function and pass the process id and locks as arguments ``` ## RL： @@ -320,7 +320,7 @@ for p in range(7): #loop over the processes https://github.com/NoteDancing/Note-documentation/blob/Note-7.0/Note%207.0%20documentation/RL/neural%20network/tensorflow/parallel/DQN.py -### PO2: +### PO1: ```python import Note.RL.parallel.kernel as k #import kernel module import DQN as d #import deep Q-network module @@ -331,15 +331,14 @@ manager=Manager() #create manager object to share data among processe kernel.init(manager) #initialize shared data with the manager kernel.action_count=2 #set the number of actions to 2 kernel.set_up(epsilon=0.01,pool_size=10000,batch=64,update_step=10) #set up the hyperparameters for training -kernel.PO=2 #use PO2 algorithm for parallel optimization +kernel.PO=1 #use PO1 algorithm for parallel optimization pool_lock=[Lock(),Lock(),Lock(),Lock(),Lock()] #create a list of locks for each process's replay pool lock=[Lock(),Lock(),Lock()] #create a list of locks for synchronization -g_lock=Lock() #create a global lock for gradient computing for p in range(5): #loop over the processes - Process(target=kernel.train,args=(p,100,lock,pool_lock,g_lock)).start() #start each process with the train function and pass the process id, the number of episodes, the locks, the pool locks and the global lock as arguments + Process(target=kernel.train,args=(p,100,lock,pool_lock)).start() #start each process with the train function and pass the process id, the number of episodes, the locks and the pool locks as arguments ``` -### PO3: +### PO2: ```python import Note.RL.parallel.kernel as k #import kernel module import DQN as d #import deep Q-network module @@ -350,12 +349,15 @@ manager=Manager() #create manager object to share data among processe kernel.init(manager) #initialize shared data with the manager kernel.action_count=2 #set the number of actions to 2 kernel.set_up(epsilon=0.01,pool_size=10000,batch=64,update_step=10) #set up the hyperparameters for training -kernel.PO=3 #use PO3 algorithm for parallel optimization +kernel.PO=2 #use PO2 algorithm for parallel optimization pool_lock=[Lock(),Lock(),Lock(),Lock(),Lock()] #create a list of locks for each process's replay pool -lock=[Lock(),Lock()] #create two locks for synchronization +lock=[Lock(),Lock(),Lock()] #create a list of locks for synchronization +g_lock=Lock() #create a global lock for gradient computing for p in range(5): #loop over the processes - Process(target=kernel.train,args=(p,100,lock,pool_lock)).start() #start each process with the train function and pass the process id, the number of episodes, the locks and the pool locks as arguments + Process(target=kernel.train,args=(p,100,lock,pool_lock,g_lock)).start() #start each process with the train function and pass the process id, the number of episodes, the locks, the pool locks and the global lock as arguments ``` + +### PO3: ```python import Note.RL.parallel.kernel as k #import kernel module import DQN as d #import deep Q-network module @@ -368,7 +370,7 @@ kernel.action_count=2 #set the number of actions to 2 kernel.set_up(epsilon=0.01,pool_size=10000,batch=64,update_step=10) #set up the hyperparameters for training kernel.PO=3 #use PO3 algorithm for parallel optimization pool_lock=[Lock(),Lock(),Lock(),Lock(),Lock()] #create a list of locks for each process's replay pool -lock=[Lock(),Lock(),Lock()] #create a list of locks for synchronization +lock=[Lock(),Lock(),Lock()] #create three locks for synchronization for p in range(5): #loop over the processes Process(target=kernel.train,args=(p,100,lock,pool_lock)).start() #start each process with the train function and pass the process id, the number of episodes, the locks and the pool locks as arguments ``` From 02dc7f3b1b85d2fe202616df97fcf92680702a74 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 27 Jul 2023 20:55:52 +0800 Subject: [PATCH 128/337] Update kernel.py --- .../RL/kernel/nspn/kernel.py | 1028 ++++++++--------- 1 file changed, 477 insertions(+), 551 deletions(-) diff --git a/Note 7.0 documentation/RL/kernel/nspn/kernel.py b/Note 7.0 documentation/RL/kernel/nspn/kernel.py index e36f68e2..72c2c679 100644 --- a/Note 7.0 documentation/RL/kernel/nspn/kernel.py +++ b/Note 7.0 documentation/RL/kernel/nspn/kernel.py @@ -4,583 +4,509 @@ import time +# This is a class for kernel-based reinforcement learning class kernel: - # This is the constructor method for initializing the class attributes + # This is the constructor method def __init__(self,nn=None,save_episode=False): - self.nn=nn # This is the neural network object - if hasattr(self.nn,'km'): # This is a condition to check if the model has a km attribute - self.nn.km=1 # This is to assign the value 1 to the km attribute - self.platform=None # This is the platform for running the algorithm, such as TensorFlow or PyTorch - self.state_pool=None # This is a pool for storing the states from the environment - self.action_pool=None # This is a pool for storing the actions taken by the agent - self.next_state_pool=None # This is a pool for storing the next states from the environment - self.reward_pool=None # This is a pool for storing the rewards received by the agent - self.done_pool=None # This is a pool for storing the done flags indicating the end of an episode - self.episode_set=[] # This is a list for storing the episodes generated by the agent - self.epsilon=None # This is a parameter for controlling the exploration-exploitation trade-off - self.episode_step=None # This is a parameter for limiting the number of steps per episode - self.pool_size=None # This is a parameter for limiting the size of the pools - self.batch=None # This is a parameter for determining the batch size for training - self.update_step=None # This is a parameter for determining the frequency of updating the model parameters - self.trial_count=None # This is a parameter for determining the number of trials for evaluating the performance - self.criterion=None # This is a parameter for determining the criterion for stopping the training - self.reward_list=[] # This is a list for storing the rewards per episode - self.suspend=False # This is a flag for indicating whether to suspend the training or not - self.save_epi=None # This is a parameter for determining whether to save the episodes or not - self.max_episode_count=None # This is a parameter for limiting the number of episodes to save - self.save_episode=save_episode # This is an argument for setting the save_epi parameter - self.filename='save.dat' # This is a filename for saving or loading the data - self.loss=None # This is a variable for storing the loss value per batch - self.loss_list=[] # This is a list for storing the loss values per episode - self.sc=0 # This is a counter for counting the number of steps taken by the agent - self.total_episode=0 # This is a counter for counting the number of episodes completed by the agent - self.time=0 # This is a variable for measuring the time elapsed per episode - self.total_time=0 # This is a variable for measuring the total time elapsed + self.nn=nn # This is the neural network model + if hasattr(self.nn,'km'): # If the model has a kernel matrix attribute + self.nn.km=1 # Set it to 1 + self.platform=None # This is the platform for the model, such as tensorflow or pytorch + self.state_pool=None # This is the pool for storing states + self.action_pool=None # This is the pool for storing actions + self.next_state_pool=None # This is the pool for storing next states + self.reward_pool=None # This is the pool for storing rewards + self.done_pool=None # This is the pool for storing done flags + self.episode_set=[] # This is the list for storing episodes + self.epsilon=None # This is the epsilon value for epsilon-greedy policy + self.episode_step=None # This is the maximum number of steps per episode + self.pool_size=None # This is the maximum size of the pool + self.batch=None # This is the batch size for training + self.update_step=None # This is the frequency of updating the network parameters + self.trial_count=None # This is the number of trials for calculating average reward + self.criterion=None # This is the criterion for stopping training when average reward reaches it + self.reward_list=[] # This is the list for storing rewards per episode + self.suspend=False # This is a flag for suspending training + self.save_epi=None # This is a flag for saving episodes + self.max_episode_count=None # This is the maximum number of episodes to save + self.save_episode=save_episode # This is a flag for saving episodes or not + self.filename='save.dat' # This is the filename for saving data + self.loss=None # This is the loss value for training + self.loss_list=[] # This is the list for storing loss values per episode + self.sc=0 # This is a counter for steps + self.total_episode=0 # This is a counter for episodes + self.time=0 # This is a timer for training time + self.total_time=0 # This is a timer for total time - # This is a method for creating an action vector based on epsilon-greedy policy - def action_vec(self): - if self.epsilon!=None: # This is a condition to check if epsilon is defined or not - self.action_one=np.ones(self.action_count,dtype=np.int8) # This is to create an array of ones with length equal to action_count, which is the number of possible actions in the environment + + def action_vec(self): # This is a method for creating action probability vector + if self.epsilon!=None: # If epsilon value is not None + self.action_one=np.ones(self.action_count,dtype=np.int8) # Create a vector of ones with length equal to action count return - # This is a method for initializing or resetting some of the class attributes - def init(self): - try: - if hasattr(self.nn,'pr'): # This is a condition to check if the model has a pr attribute, which stands for prioritized replay buffer - self.nn.pr.TD=np.array(0) # This is to assign an array of zero to the TD attribute, which stands for temporal difference error - except Exception as e: # This is to catch any exception that may occur during this process - raise e # This is to raise or re-throw the exception - self.suspend=False # This is to reset the suspend flag to False - self.save_epi=None # This is to reset the save_epi parameter to None - self.episode_set=[] # This is to reset the episode_set list to an empty list - self.state_pool=None # This is to reset the state_pool to None - self.action_pool=None # This is to reset the action_pool to None - self.next_state_pool=None # This is to reset the next_state_pool to None - self.reward_pool=None # This is to reset the reward_pool to None - self.done_pool=None # This is to reset the done_pool to None - self.reward_list=[] # This is to reset the reward_list to an empty list - self.loss=0 # This is to reset the loss value to zero - self.loss_list=[] # This is to reset the loss_list to an empty list - self.sc=0 # This is to reset the step counter to zero - self.total_episode=0 # This is to reset the episode counter to zero - self.time=0 # This is to reset the time variable to zero - self.total_time=0 # This is to reset the total_time variable to zero - return + def init(self): # This is a method for initializing some attributes + try: # Try to execute the following code block + if hasattr(self.nn,'pr'): # If the model has a priority replay attribute + self.nn.pr.TD=np.array(0) # Set it to zero array + except Exception as e: # If an exception occurs + raise e # Raise the exception and exit the method + self.suspend=False # Set the suspend flag to False (not suspending training) + self.save_epi=None # Set the save episode flag to None (not saving episodes) + self.episode_set=[] # Initialize the episode set list to empty list (no episodes saved) + self.state_pool=None # Set the state pool to None (no states stored) + self.action_pool=None # Set the action pool to None (no actions stored) + self.next_state_pool=None # Set the next state pool to None (no next states stored) + self.reward_pool=None # Set the reward pool to None (no rewards stored) + self.done_pool=None # Set the done pool to None (no done flags stored) + self.reward_list=[] # Initialize the reward list to empty list (no rewards recorded) + self.loss=0 # Set the loss value to zero (no loss calculated) + self.loss_list=[] # Initialize the loss list to empty list (no losses recorded) + self.sc=0 # Set the step counter to zero (no steps taken) + self.total_episode=0 # Set the total episode counter to zero (no episodes completed) + self.time=0 # Set the time value to zero (no time elapsed) + self.total_time=0 # Set the total time value to zero (no total time elapsed) + return # Return nothing and exit the method - # This is a method for setting up some of the class attributes based on the arguments - def set_up(self,epsilon=None,episode_step=None,pool_size=None,batch=None,update_step=None,trial_count=None,criterion=None): - if epsilon!=None: # This is a condition to check if epsilon is given or not - self.epsilon=epsilon # This is to assign the epsilon value to the epsilon attribute - if episode_step!=None: # This is a condition to check if episode_step is given or not - self.episode_step=episode_step # This is to assign the episode_step value to the episode_step attribute - if pool_size!=None: # This is a condition to check if pool_size is given or not - self.pool_size=pool_size # This is to assign the pool_size value to the pool_size attribute - if batch!=None: # This is a condition to check if batch is given or not - self.batch=batch # This is to assign the batch value to the batch attribute - if update_step!=None: # This is a condition to check if update_step is given or not - self.update_step=update_step # This is to assign the update_step value to the update_step attribute - if trial_count!=None: # This is a condition to check if trial_count is given or not - self.trial_count=trial_count # This is to assign the trial_count value to the trial_count attribute - if criterion!=None: # This is a condition to check if criterion is given or not - self.criterion=criterion # This is to assign the criterion value to the criterion attribute - self.action_vec() # This is to call the action_vec method for creating an action vector based on epsilon-greedy policy + def set_up(self,epsilon=None,episode_step=None,pool_size=None,batch=None,update_step=None,trial_count=None,criterion=None): # This is a method for setting up some parameters + if epsilon!=None: # If epsilon value is given + self.epsilon=epsilon # Set it to the given value + if episode_step!=None: # If episode step value is given + self.episode_step=episode_step # Set it to the given value + if pool_size!=None: # If pool size value is given + self.pool_size=pool_size # Set it to the given value + if batch!=None: # If batch size value is given + self.batch=batch # Set it to the given value + if update_step!=None: # If update step value is given + self.update_step=update_step # Set it to the given value + if trial_count!=None: # If trial count value is given + self.trial_count=trial_count # Set it to the given value + if criterion!=None: # If criterion value is given + self.criterion=criterion # Set it to the given value + self.action_vec() # Call the action vector method return - # This is a method for implementing an epsilon-greedy policy for choosing an action based on a state - def epsilon_greedy_policy(self,s): - action_prob=self.action_one*self.epsilon/len(self.action_one) # This is to create an array of probabilities for each action based on epsilon and action_one, which are class attributes + def epsilon_greedy_policy(self,s): # This is a method for implementing epsilon-greedy policy + action_prob=self.action_one*self.epsilon/len(self.action_one) # Create a vector of action probabilities with epsilon/number of actions for each action try: - if hasattr(self.platform,'DType'): # This is a condition to check if the platform has a DType attribute, which indicates that it is TensorFlow - best_a=np.argmax(self.nn.nn.fp(s)) # This is to find the best action based on the neural network model's output for the state s using fp method, which stands for forward propagation - action_prob[best_a]+=1-self.epsilon # This is to increase the probability of choosing the best action by 1-epsilon - else: # This means that the platform does not have a DType attribute, which indicates that it is PyTorch - s=self.platform.tensor(s,dtype=self.platform.float).to(self.nn.device) # This is to convert the state s into a PyTorch tensor with float data type and send it to the device where the model runs, such as CPU or GPU - best_a=self.nn.nn(s).argmax() # This is to find the best action based on the neural network model's output for the state s using nn method, which stands for neural network - action_prob[best_a.numpy()]+=1-self.epsilon # This is to increase the probability of choosing the best action by 1-epsilon and convert it into a numpy array - except Exception as e: # This is to catch any exception that may occur during this process - raise e # This is to raise or re-throw the exception - return action_prob + if hasattr(self.platform,'DType'): # If the platform is tensorflow + best_a=np.argmax(self.nn.nn.fp(s)) # Get the best action by using the network's forward propagation method on the state + action_prob[best_a]+=1-self.epsilon # Add 1-epsilon to the best action's probability + else: # If the platform is pytorch + s=self.platform.tensor(s,dtype=self.platform.float).to(self.nn.device) # Convert the state to a tensor with float type and send it to the device (cpu or gpu) + best_a=self.nn.nn(s).argmax() # Get the best action by using the network's forward method on the state + action_prob[best_a.numpy()]+=1-self.epsilon # Add 1-epsilon to the best action's probability and convert it to numpy array + except Exception as e: + raise e + return action_prob # Return the action probability vector - # This is a method for computing the loss and updating the model parameters using TensorFlow - @tf.function(jit_compile=True) - def tf_opt(self,state_batch,action_batch,next_state_batch,reward_batch,done_batch): - with self.platform.GradientTape(persistent=True) as tape: # This is to create a gradient tape object for recording and computing gradients in TensorFlow - loss=self.nn.loss(state_batch,action_batch,next_state_batch,reward_batch,done_batch) # This is to compute the loss value based on the model's loss function and the data batches - if hasattr(self.nn,'gradient'): # This is a condition to check if the model has a gradient attribute, which stands for a custom gradient function - gradient=self.nn.gradient(tape,loss) # This is to compute the gradients based on the model's gradient function and the tape and loss objects - if hasattr(self.nn.opt,'apply_gradients'): # This is a condition to check if the model's optimizer has an apply_gradients method, which indicates that it is a TensorFlow optimizer - self.nn.opt.apply_gradients(zip(gradient,self.nn.param)) # This is to apply the gradients to the model's parameters using the optimizer's apply_gradients method - else: # This means that the model's optimizer does not have an apply_gradients method, which indicates that it is a custom optimizer function - self.nn.opt(gradient) # This is to update the model's parameters using the optimizer function and the gradients - else: # This means that the model does not have a gradient attribute, which indicates that it uses the default gradient computation - if hasattr(self.nn,'nn'): # This is a condition to check if the model has a nn attribute, which stands for a single neural network model - gradient=tape.gradient(loss,self.nn.param) # This is to compute the gradients based on the tape and loss objects and the model's parameters - self.nn.opt.apply_gradients(zip(gradient,self.nn.param)) # This is to apply the gradients to the model's parameters using the optimizer's apply_gradients method - else: # This means that the model does not have a nn attribute, which indicates that it uses two neural network models, such as actor and critic - actor_gradient=tape.gradient(loss[0],self.nn.param[0]) # This is to compute the gradients for the actor model based on the first element of the loss object and the actor model's parameters - critic_gradient=tape.gradient(loss[1],self.nn.param[1]) # This is to compute the gradients for the critic model based on the second element of the loss object and the critic model's parameters - self.nn.opt.apply_gradients(zip(actor_gradient,self.nn.param[0])) # This is to apply the gradients to the actor model's parameters using the optimizer's apply_gradients method - self.nn.opt.apply_gradients(zip(critic_gradient,self.nn.param[1])) # This is to apply the gradients to the critic model's parameters using the optimizer's apply_gradients method - return loss # This is to return the loss value + def opt(self,state_batch,action_batch,next_state_batch,reward_batch,done_batch): # This is a method for optimizing the network parameters with a batch of data + if hasattr(self.platform,'DType'): # If the platform is tensorflow + loss=self.tf_opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch) # Call the tensorflow optimization method with the data batch and get the loss value + else: # If the platform is pytorch + loss=self.pytorch_opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch) # Call the pytorch optimization method with the data batch and get the loss value + return loss # Return the loss value - # This is a method for computing the loss and updating the model parameters using PyTorch - def pytorch_opt(self,state_batch,action_batch,next_state_batch,reward_batch,done_batch): - loss=self.nn.loss(state_batch,action_batch,next_state_batch,reward_batch,done_batch) # This is to compute the loss value based on the model's loss function and the data batches - self.nn.backward(loss) # This is to compute and accumulate the gradients based on the model's backward method and the loss object - self.nn.opt() # This is to update and reset the model's parameters based on the model's opt method, which stands for optimizer function - return loss # This is to return the loss value + def opt_ol(self,state,action,next_state,reward,done): # This is a method for optimizing the network parameters with online data (one sample) + if hasattr(self.platform,'DType'): # If the platform is tensorflow + loss=self.tf_opt(state,action,next_state,reward,done) # Call the tensorflow optimization method with the online data and get the loss value + else: # If the platform is pytorch + loss=self.pytorch_opt(state,action,next_state,reward,done) # Call the pytorch optimization method with the online data and get the loss value + return loss # Return the loss value - # This is a method for choosing between tf_opt or pytorch_opt based on the platform attribute - def opt(self,state_batch,action_batch,next_state_batch,reward_batch,done_batch): - if hasattr(self.platform,'DType'): # This is a condition to check if the platform has a DType attribute, which indicates that it is TensorFlow - loss=self.tf_opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch) # This is to call the tf_opt method and pass the data batches as arguments - else: # This means that the platform does not have a DType attribute, which indicates that it is PyTorch - loss=self.pytorch_opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch) # This is to call the pytorch_opt method and pass the data batches as arguments - return loss # This is to return the loss value + def pool(self,s,a,next_s,r,done): # This is a method for storing data in the pool + if type(self.state_pool)!=np.ndarray and self.state_pool==None: # If this is the first time to store data in the pool (the pool is None) + self.state_pool=s # Set the state pool to be equal to s (the state) + if type(a)==int: # If a (the action) is an integer (discrete action space) + a=np.array(a) # Convert it to a numpy array + self.action_pool=np.expand_dims(a,axis=0) # Expand its dimension by one and set it as the action pool + else: # If a (the action) is not an integer (continuous action space) + self.action_pool=a # Set it as the action pool + self.next_state_pool=np.expand_dims(next_s,axis=0) # Expand the dimension of next_s (the next state) by one and set it as the next state pool + self.reward_pool=np.expand_dims(r,axis=0) # Expand the dimension of r (the reward) by one and set it as the reward pool + self.done_pool=np.expand_dims(done,axis=0) # Expand the dimension of done (the done flag) by one and set it as the done pool + else: # If this is not the first time to store data in the pool (the pool is not None) + self.state_pool=np.concatenate((self.state_pool,s),0) # Concatenate s (the state) with the state pool along the first axis + if type(a)==int: # If a (the action) is an integer (discrete action space) + a=np.array(a) # Convert it to a numpy array + self.action_pool=np.concatenate((self.action_pool,np.expand_dims(a,axis=0)),0) # Expand its dimension by one and concatenate it with the action pool along the first axis + else: # If a (the action) is not an integer (continuous action space) + self.action_pool=np.concatenate((self.action_pool,a),0) # Concatenate a (the action) with the action pool along the first axis + self.next_state_pool=np.concatenate((self.next_state_pool,np.expand_dims(next_s,axis=0)),0) # Expand the dimension of next_s (the next state) by one and concatenate it with the next state pool along the first axis + self.reward_pool=np.concatenate((self.reward_pool,np.expand_dims(r,axis=0)),0) # Expand the dimension of r (the reward) by one and concatenate it with the reward pool along the first axis + self.done_pool=np.concatenate((self.done_pool,np.expand_dims(done,axis=0)),0) # Expand the dimension of done (the done flag) by one and concatenate it with the done pool along the first axis + if len(self.state_pool)>self.pool_size: # If the length of the state pool exceeds the maximum size of the pool + self.state_pool=self.state_pool[1:] # Remove the first element of the state pool + self.action_pool=self.action_pool[1:] # Remove the first element of the action pool + self.next_state_pool=self.next_state_pool[1:] # Remove the first element of the next state pool + self.reward_pool=self.reward_pool[1:] # Remove the first element of the reward pool + self.done_pool=self.done_pool[1:] # Remove the first element of the done pool + return # Return nothing - # This is a method for computing the loss and updating the model parameters using online data - def opt_ol(self,state,action,next_state,reward,done): - if hasattr(self.platform,'DType'): # This is a condition to check if the platform has a DType attribute, which indicates that it is TensorFlow - loss=self.tf_opt(state,action,next_state,reward,done) # This is to call the tf_opt method and pass the online data as arguments - else: # This means that the platform does not have a DType attribute, which indicates that it is PyTorch - loss=self.pytorch_opt(state,action,next_state,reward,done) # This is to call the pytorch_opt method and pass the online data as arguments - return loss # This is to return the loss value + def choose_action(self,s): # This is a method for choosing an action based on a state + if hasattr(self.nn,'nn'): # If the model has a network attribute + if hasattr(self.platform,'DType'): # If the platform is tensorflow + if self.epsilon==None: # If epsilon value is None + self.epsilon=self.nn.epsilon(self.sc) # Set it to be equal to the model's epsilon method with sc (step counter) as input + action_prob=self.epsilon_greedy_policy(s) # Get the action probability vector by using epsilon-greedy policy method with s (state) as input + a=np.random.choice(self.action_count,p=action_prob) # Choose an action randomly according to the action probability vector + else: # If the platform is pytorch + if self.epsilon==None: # If epsilon value is None + self.epsilon=self.nn.epsilon(self.sc) # Set it to be equal to the model's epsilon method with sc (step counter) as input + action_prob=self.epsilon_greedy_policy(s) # Get the action probability vector by using epsilon-greedy policy method with s (state) as input + a=np.random.choice(self.action_count,p=action_prob.numpy()) # Choose an action randomly according to the action probability vector converted to numpy array + else: # If the model does not have a network attribute + if hasattr(self.nn,'action'): # If the model has an action attribute + if hasattr(self.platform,'DType'): # If the platform is tensorflow + if self.epsilon==None: # If epsilon value is None + self.epsilon=self.nn.epsilon(self.sc) # Set it to be equal to the model's epsilon method with sc (step counter) as input + if hasattr(self.nn,'discriminator'): # If the model has a discriminator attribute + a=self.nn.action(s) # Get an action by using model's action method with s (state) as input + reward=self.nn.discriminator(s,a) # Get a reward by using model's discriminator method with s (state) and a (action) as inputs + s=np.squeeze(s) # Squeeze s (state) to remove redundant + else: # If the model does not have a discriminator attribute + a=self.nn.action(s).numpy() # Get an action by using model's action method with s (state) as input and convert it to numpy array + else: # If the platform is pytorch + if self.epsilon==None: # If epsilon value is None + self.epsilon=self.nn.epsilon(self.sc) # Set it to be equal to the model's epsilon method with sc (step counter) as input + if hasattr(self.nn,'discriminator'): # If the model has a discriminator attribute + a=self.nn.action(s) # Get an action by using model's action method with s (state) as input + reward=self.nn.discriminator(s,a) # Get a reward by using model's discriminator method with s (state) and a (action) as inputs + s=np.squeeze(s) # Squeeze s (state) to remove redundant dimensions + else: # If the model does not have a discriminator attribute + a=self.nn.action(s).detach().numpy() # Get an action by using model's action method with s (state) as input and convert it to numpy array + else: # If the model does not have an action attribute + if hasattr(self.platform,'DType'): # If the platform is tensorflow + a=(self.nn.actor.fp(s)+self.nn.noise()).numpy() # Get an action by using model's actor network's forward propagation method with s (state) and noise as inputs and convert it to numpy array + else: # If the platform is pytorch + s=self.platform.tensor(s,dtype=self.platform.float).to(self.nn.device) # Convert the state to a tensor with float type and send it to the device (cpu or gpu) + a=(self.nn.actor(s)+self.nn.noise()).detach().numpy() # Get an action by using model's actor network's forward method with s (state) and noise as inputs and convert it to numpy array + if hasattr(self.nn,'discriminator'): # If the model has a discriminator attribute + return a,reward # Return the action and the reward + else: # If the model does not have a discriminator attribute + return a,None # Return the action and None - # This is a method for storing the data into the pools - def pool(self,s,a,next_s,r,done): - if type(self.state_pool)!=np.ndarray and self.state_pool==None: # This is a condition to check if the state_pool is not an array and is None, which indicates that it is empty - self.state_pool=s # This is to assign the state s to the state_pool - if type(a)==int: # This is a condition to check if the action a is an integer, which indicates that it is a discrete action - a=np.array(a) # This is to convert the action a into an array - self.action_pool=np.expand_dims(a,axis=0) # This is to expand the dimension of the action array and assign it to the action_pool - else: # This means that the action a is not an integer, which indicates that it is a continuous action - self.action_pool=a # This is to assign the action a to the action_pool - self.next_state_pool=np.expand_dims(next_s,axis=0) # This is to expand the dimension of the next state next_s and assign it to the next_state_pool - self.reward_pool=np.expand_dims(r,axis=0) # This is to expand the dimension of the reward r and assign it to the reward_pool - self.done_pool=np.expand_dims(done,axis=0) # This is to expand the dimension of the done flag done and assign it to the done_pool - else: # This means that the state_pool is not empty - self.state_pool=np.concatenate((self.state_pool,s),0) # This is to concatenate or append the state s to the state_pool along axis 0 - if type(a)==int: # This is a condition to check if the action a is an integer, which indicates that it is a discrete action - a=np.array(a) # This is to convert the action a into an array - self.action_pool=np.concatenate((self.action_pool,np.expand_dims(a,axis=0)),0) # This is to expand the dimension of the action array and concatenate or append it to the action_pool along axis 0 - else: # This means that the action a is not an integer, which indicates that it is a continuous action - self.action_pool=np.concatenate((self.action_pool,a),0) # This is to concatenate or append the action a to the action_pool along axis 0 - self.next_state_pool=np.concatenate((self.next_state_pool,np.expand_dims(next_s,axis=0)),0) # This is to expand the dimension of the next state next_s and concatenate or append it to the next_state_pool along axis 0 - self.reward_pool=np.concatenate((self.reward_pool,np.expand_dims(r,axis=0)),0) # This is to expand the dimension of the reward r and concatenate or append it to the reward_pool along axis 0 - self.done_pool=np.concatenate((self.done_pool,np.expand_dims(done,axis=0)),0) # This is to expand the dimension of the done flag done and concatenate or append it to the done_pool along axis 0 - if len(self.state_pool)>self.pool_size: # This is a condition to check if the length of the state_pool exceeds the pool_size attribute, which indicates that it needs trimming - self.state_pool=self.state_pool[1:] # This is to remove or pop out the first element of the state_pool - self.action_pool=self.action_pool[1:] # This is to remove or pop out the first element of the action_pool - self.next_state_pool=self.next_state_pool[1:] # This is to remove or pop out the first element of the next_state_pool - self.reward_pool=self.reward_pool[1:] # This is to remove or pop out the first element of the reward_pool - self.done_pool=self.done_pool[1:] # This is to remove or pop out the first element of the done pool - return - - - # This is a method for training the model using data from pools - def _train(self): - if len(self.state_pool)self.pool_size: # This is a condition to check if the length of the state_pool exceeds the pool_size attribute, which indicates that it needs trimming - TD=np.array(0) # This is to create an array with zero value as TD - self.nn.pr.TD=np.append(TD,self.nn.pr.TD[2:]) # This is to append TD to the TD attribute of pr after removing or popping out the first two elements - self.reward=r+self.reward # This is to update the reward value by adding r to it - loss=self._train() # This is to call the _train method for training the model using data from pools and return the loss value - self.sc+=1 # This is to increase the step counter by one - if done: # This is a condition to check if the done flag is True, which indicates that the episode has ended - if self.save_episode==True: # This is a condition to check if the save_episode attribute is True, which indicates that it needs to save an episode - episode=[s,a,next_s,r] # This is to assign a list of state s, action a, next state next_s and reward r as an episode - self.reward_list.append(self.reward) # This is to append the reward value to the reward_list attribute, which stands for a list of rewards per episode - return loss,episode,done # This is to return the loss value, episode and done flag as outputs - elif self.save_episode==True: # This is a condition to check if the save_episode attribute is True, which indicates that it needs to save an episode - episode=[s,a,next_s,r] # This is to assign a list of state s, action a, next state next_s and reward r as an episode - s=next_s # This is to update the state s by assigning next_s to it - else: # This is an else clause for the case when the episode_step attribute is not None, which indicates that there is a limit on the number of steps per episode - for _ in range(self.episode_step): # This is a loop for running an episode for a fixed number of steps - self.suspend_func() # This is to call the suspend_func method for suspending the training if needed - if hasattr(self.nn,'nn'): # This is a condition to check if the model has a nn attribute, which stands for a single neural network model - if hasattr(self.platform,'DType'): # This is a condition to check if the platform has a DType attribute, which indicates that it is TensorFlow - s=np.expand_dims(s,axis=0) # This is to expand the dimension of the state s along axis 0 - if self.epsilon==None: # This is a condition to check if the epsilon attribute is None, which indicates that it needs to be initialized - self.epsilon=self.nn.epsilon(self.sc) # This is to assign the epsilon value returned by the model's epsilon method, which stands for epsilon function, to the epsilon attribute - action_prob=self.epsilon_greedy_policy(s) # This is to call the epsilon_greedy_policy method for choosing an action based on a state and return its probability - a=np.random.choice(self.action_count,p=action_prob) # This is to sample an action from a discrete distribution with probabilities given by action_prob - else: # This means that the platform does not have a DType attribute, which indicates that it is PyTorch - s=np.expand_dims(s,axis=0) # This is to expand the dimension of the state s along axis 0 - if self.epsilon==None: # This is a condition to check if the epsilon attribute is None, which indicates that it needs to be initialized - self.epsilon=self.nn.epsilon(self.sc) # This is to assign the epsilon value returned by the model's epsilon method, which stands for epsilon function, to the epsilon attribute - action_prob=self.epsilon_greedy_policy(s) # This is to call the epsilon_greedy_policy method for choosing an action based on a state and return its probability - a=np.random.choice(self.action_count,p=action_prob) # This is to sample an action from a discrete distribution with probabilities given by action_prob - else: # This means that the model does not have a nn attribute, which indicates that it uses two neural network models, such as actor and critic - if hasattr(self.platform,'DType'): # This is a condition to check if the platform has a DType attribute, which indicates that it is TensorFlow - s=np.expand_dims(s,axis=0) # This is to expand the dimension of the state s along axis 0 - if self.epsilon==None: # This is a condition to check if the epsilon attribute is None, which indicates that it needs to be initialized - self.epsilon=self.nn.epsilon(self.sc) # This is to assign the epsilon value returned by the model's epsilon method, which stands for epsilon function, to the epsilon attribute - if hasattr(self.nn,'discriminator'): # This is a condition to check if the model has a discriminator attribute, which stands for a discriminator model that evaluates the quality of an action - a=self.nn.action(s) # This is to get an action from the model's action method, which stands for action function, based on the state s - reward=self.nn.discriminator(s,a) # This is to get a reward from the model's discriminator method, which stands for discriminator function, based on the state s and action a - s=np.squeeze(s) # This is to remove any singleton dimensions from the state s - else: # This means that the model does not have a discriminator attribute, which indicates that it uses the environment's reward function - a=self.nn.action(s).numpy() # This is to get an action from the model's action method, which stands for action function, based on the state s and convert it into a numpy array - else: # This means that the platform does not have a DType attribute, which indicates that it is PyTorch - s=np.expand_dims(s,axis=0) # This is to expand the dimension of the state s along axis 0 - if self.epsilon==None: # This is a condition to check if the epsilon attribute is None, which indicates that it needs to be initialized - self.epsilon=self.nn.epsilon(self.sc) # This is to assign the epsilon value returned by the model's epsilon method, which stands for epsilon function, to the epsilon attribute - if hasattr(self.nn,'discriminator'): # This is a condition to check if the model has a discriminator attribute, which stands for a discriminator model that evaluates the quality of an action - a=self.nn.action(s) # This is to get an action from the model's action method, which stands for action function, based on the state s - reward=self.nn.discriminator(s,a) # This is to get a reward from the model's discriminator method, which stands for discriminator function, based on the state s and action a - s=np.squeeze(s) # This is to remove any singleton dimensions from the state s - else: # This means that the model does not have a discriminator attribute, which indicates that it uses the environment's reward function - a=self.nn.action(s).detach().numpy() # This is to get an action from the model's action method, which stands for action function, based on the state s and detach it from the computation graph and convert it into a numpy array - next_s,r,done=self.nn.env(a) # This is to get the next state next_s, reward r and done flag done from the environment using the model's env method based on the action a - if hasattr(self.platform,'DType'): # This is a condition to check if the platform has a DType attribute, which indicates that it is TensorFlow - if type(self.nn.param[0])!=list: # This is a condition to check if the first element of the model's param attribute, which stands for parameters, is not a list, which indicates that it has only one neural network model - next_s=np.array(next_s,self.nn.param[0].dtype.name) # This is to convert the next state next_s into an array with data type matching with the model's parameter data type - r=np.array(r,self.nn.param[0].dtype.name) # This is to convert the reward r into an array with data type matching with the model's parameter data type - done=np.array(done,self.nn.param[0].dtype.name) # This is to convert the done flag done into an array with data type matching with the model's parameter data type - else: # This means that the first element of the model's param attribute is a list, which indicates that it has two neural network models, such as actor and critic - next_s=np.array(next_s,self.nn.param[0][0].dtype.name) # This is to convert the next state next_s into an array with data type matching with the first model’s parameter data type - r=np.array(r,self.nn.param[0][0].dtype.name) # This is to convert the reward r into an array with data type matching with the first model's parameter data type - done=np.array(done,self.nn.param[0][0].dtype.name) # This is to convert the done flag done into an array with data type matching with the first model's parameter data type - if hasattr(self.nn,'pool'): # This is a condition to check if the model has a pool attribute, which stands for a custom pool function - if hasattr(self.nn,'discriminator'): # This is a condition to check if the model has a discriminator attribute, which stands for a discriminator model that evaluates the quality of an action - self.nn.pool(self.state_pool,self.action_pool,self.next_state_pool,self.reward_pool,self.done_pool,[s,a,next_s,reward,done]) # This is to call the model's pool method for storing the data into the pools using the state s, action a, next state next_s, reward reward and done flag done - else: # This means that the model does not have a discriminator attribute, which indicates that it uses the environment's reward function - self.nn.pool(self.state_pool,self.action_pool,self.next_state_pool,self.reward_pool,self.done_pool,[s,a,next_s,r,done]) # This is to call the model's pool method for storing the data into the pools using the state s, action a, next state next_s, reward r and done flag done - else: # This means that the model does not have a pool attribute, which indicates that it uses the default pool method - self.pool(s,a,next_s,r,done) # This is to call the self.pool method for storing the data into the pools using the state s, action a, next state next_s, reward r and done flag done - if hasattr(self.nn,'pr'): # This is a condition to check if the model has a pr attribute, which stands for priority replay - self.nn.pr.TD=np.append(self.nn.pr.TD,self.nn.initial_TD) # This is to append the initial TD value returned by the model's initial_TD method, which stands for initial temporal difference function, to the TD attribute of pr, which stands for temporal difference values - if len(self.state_pool)>self.pool_size: # This is a condition to check if the length of the state_pool exceeds the pool_size attribute, which indicates that it needs trimming - TD=np.array(0) # This is to create an array with zero value as TD - self.nn.pr.TD=np.append(TD,self.nn.pr.TD[2:]) # This is to append TD to the TD attribute of pr after removing or popping out the first two elements - self.reward=r+self.reward # This is to update the reward value by adding r to it - loss=self._train() # This is to call the _train method for training the model using data from pools and return the loss value - self.sc+=1 # This is to increase the step counter by one - if done: # This is a condition to check if the done flag is True, which indicates that the episode has ended - if self.save_episode==True: # This is a condition to check if the save_episode attribute is True, which indicates that it needs to save an episode - episode=[s,a,next_s,r] # This is to assign a list of state s, action a, next state next_s and reward r as an episode - self.reward_list.append(self.reward) # This is to append the reward value to the reward_list attribute, which stands for a list of rewards per episode - return loss,episode,done # This is to return the loss value, episode and done flag as outputs - elif self.save_episode==True: # This is a condition to check if the save_episode attribute is True, which indicates that it needs to save an episode - episode=[s,a,next_s,r] # This is to assign a list of state s, action a, next state next_s and reward r as an episode - s=next_s # This is to update the state s by assigning next_s to it - self.reward_list.append(self.reward) # This is to append the reward value to the reward_list attribute, which stands for a list of rewards per episode - return loss,episode,done # This is to return the loss value, episode and done flag as outputs - + def train_(self): # This is a method for training the network for one episode + episode=[] # Initialize episode list to empty list + self.reward=0 # Initialize reward value to zero + s=self.nn.env(initial=True) # Get an initial state from the model's environment method with initial flag set to True + if hasattr(self.platform,'DType'): # If the platform is tensorflow + if type(self.nn.param[0])!=list: # If the first element of model's parameter list is not a list (single network) + s=np.array(s,self.nn.param[0].dtype.name) # Convert s (state) to a numpy array with the same data type as the first element of model's parameter list + else: # If the first element of model's parameter list is a list (multiple networks) + s=np.array(s,self.nn.param[0][0].dtype.name) # Convert s (state) to a numpy array with the same data type as the first element of the first element of model's parameter list + if self.episode_step==None: # If episode step value is None (no limit on steps per episode) + while True: # Loop indefinitely until done flag is True or break statement is executed + s=np.expand_dims(s,axis=0) # Expand s (state) dimension by one + a,reward=self.choose_action(s) # Get an action and a reward by using choose action method with s (state) as input + next_s,r,done=self.nn.env(a) # Get next state, reward + if hasattr(self.platform,'DType'): # If the platform is tensorflow + if type(self.nn.param[0])!=list: # If the first element of model's parameter list is not a list (single network) + next_s=np.array(next_s,self.nn.param[0].dtype.name) # Convert next_s (next state) to a numpy array with the same data type as the first element of model's parameter list + r=np.array(r,self.nn.param[0].dtype.name) # Convert r (reward) to a numpy array with the same data type as the first element of model's parameter list + done=np.array(done,self.nn.param[0].dtype.name) # Convert done (done flag) to a numpy array with the same data type as the first element of model's parameter list + else: # If the first element of model's parameter list is a list (multiple networks) + next_s=np.array(next_s,self.nn.param[0][0].dtype.name) # Convert next_s (next state) to a numpy array with the same data type as the first element of the first element of model's parameter list + r=np.array(r,self.nn.param[0][0].dtype.name) # Convert r (reward) to a numpy array with the same data type as the first element of the first element of model's parameter list + done=np.array(done,self.nn.param[0][0].dtype.name) # Convert done (done flag) to a numpy array with the same data type as the first element of the first element of model's parameter list + if hasattr(self.nn,'pool'): # If the model has a pool attribute + if hasattr(self.nn,'discriminator'): # If the model has a discriminator attribute + self.nn.pool(self.state_pool,self.action_pool,self.next_state_pool,self.reward_pool,self.done_pool,[s,a,next_s,reward,done]) # Call the model's pool method with pools and data as inputs + else: # If the model does not have a discriminator attribute + self.nn.pool(self.state_pool,self.action_pool,self.next_state_pool,self.reward_pool,self.done_pool,[s,a,next_s,r,done]) # Call the model's pool method with pools and data as inputs + else: # If the model does not have a pool attribute + self.pool(s,a,next_s,r,done) # Call the pool method with data as inputs + if hasattr(self.nn,'pr'): # If the model has a priority replay attribute + self.nn.pr.TD=np.append(self.nn.pr.TD,self.nn.initial_TD) # Append the initial TD value to the TD array of priority replay + if len(self.state_pool)>self.pool_size: # If the length of the state pool exceeds the maximum size of the pool + TD=np.array(0) # Create a zero array for TD value + self.nn.pr.TD=np.append(TD,self.nn.pr.TD[2:]) # Append TD value to the TD array of priority replay and remove the first two elements + self.reward=r+self.reward # Add r (reward) to total reward value + loss=self._train() # Call the _train method and get loss value + self.sc+=1 # Increase sc (step counter) by one + if done: # If done flag is True + if self.save_episode==True: # If save episode flag is True + episode=[s,a,next_s,r] # Set episode list to be equal to data + self.reward_list.append(self.reward) # Append total reward value to reward list + return loss,episode,done # Return loss value, episode list, and done flag + elif self.save_episode==True: # If save episode flag is True and done flag is False + episode=[s,a,next_s,r] # Set episode list to be equal to data + s=next_s # Set s (state) to be equal to next_s (next state) + else: # If episode step value is not None (there is a limit on steps per episode) + for _ in range(self.episode_step): # For each step in episode step value + s=np.expand_dims(s,axis=0) # Expand s (state) dimension by one + a,reward=self.choose_action(s) # Get an action and a reward by using choose action method with s (state) as input + next_s,r,done=self.nn.env(a) # Get next state, reward, and done flag by using model's environment method with a (action) as input + if hasattr(self.platform,'DType'): # If the platform is tensorflow + if type(self.nn.param[0])!=list: # If the first element of model's parameter list is not a list (single network) + next_s=np.array(next_s,self.nn.param[0].dtype.name) # Convert next_s (next state) to a numpy array with the same data type as the first element of model's parameter list + r=np.array(r,self.nn.param[0].dtype.name) # Convert r (reward) to a numpy array with the same data type as the first element of model's parameter list + done=np.array(done,self.nn.param[0].dtype.name) # Convert done (done flag) to a numpy array with the same data type as the first element of model's parameter list + else: # If the first element of model's parameter list is a list (multiple networks) + next_s=np.array(next_s,self.nn.param[0][0].dtype.name) # Convert next_s (next state) to a numpy array with the same data type as the first element of the first element of model's parameter list + r=np.array(r,self.nn.param[0][0].dtype.name) # Convert r (reward) to a numpy array with the same data type as the first element of the first element of model's parameter list + done=np.array(done,self.nn.param[0][0].dtype.name) # Convert done (done flag) to a numpy array with the same data type as the first element of the first element of model's parameter list + if hasattr(self.nn,'pool'): # If the model has a pool attribute + if hasattr(self.nn,'discriminator'): # If the model has a discriminator attribute + self.nn.pool(self.state_pool,self.action_pool,self.next_state_pool,self.reward_pool,self.done_pool,[s,a,next_s,reward,done]) # Call the model's pool method with pools and data as inputs + else: # If the model does not have a discriminator attribute + self.nn.pool(self.state_pool,self.action_pool,self.next_state_pool,self.reward_pool,self.done_pool,[s,a,next_s,r,done]) # Call the model's pool method with pools and data as inputs + else: # If the model does not have a pool attribute + self.pool(s,a,next_s,r,done) # Call the pool method with data as inputs + if hasattr(self.nn,'pr'): # If the model has a priority replay attribute + self.nn.pr.TD=np.append(self.nn.pr.TD,self.nn.initial_TD) # Append the initial TD value to the TD array of priority replay + if len(self.state_pool)>self.pool_size: # If the length of the state pool exceeds the maximum size of the pool + TD=np.array(0) # Create a zero array for TD value + self.nn.pr.TD=np.append(TD,self.nn.pr.TD[2:]) # Append TD value to the TD array of priority replay and remove the first two elements + self.reward=r+self.reward # Add r (reward) to total reward value + loss=self._train() # Call the _train method and get loss value + self.sc+=1 # Increase sc (step counter) by one + if done: # If done flag is True + if self.save_episode==True: # If save episode flag is True + episode=[s,a,next_s,r] # Set episode list to be equal to data + self.reward_list.append(self.reward) # Append total reward value to reward list + return loss,episode,done # Return loss value, episode list, and done flag + elif self.save_episode==True: # If save episode flag is True and done flag is False + episode=[s,a,next_s,r] # Set episode list to be equal to data + s=next_s # Set s (state) to be equal to next_s (next state) + self.reward_list.append(self.reward) # Append total reward value to reward list + return loss,episode,done # Return loss value, episode list, and done flag + + + def train(self,episode_count,save=None,one=True,p=None,s=None): # This is a method for training the network for multiple episodes + avg_reward=None # Initialize average reward value to None + if p==None: # If p value is None (p is used for printing frequency) + self.p=9 # Set it to be 9 (print every 10 episodes) + else: # If p value is given + self.p=p-1 # Set it to be p minus one (print every p episodes) + if s==None: # If s value is None (s is used for saving frequency) + self.s=1 # Set it to be 1 (save every episode) + self.file_list=None # Set file list to be None (no need to store file names) + else: # If s value is given + self.s=s-1 # Set it to be s minus one (save every s episodes) + self.file_list=[] # Initialize file list to empty list (need to store file names) + if episode_count!=None: # If episode count value is not None (there is a limit on episodes) + for i in range(episode_count): # For each episode in episode count value + t1=time.time() # Record the start time of the episode + loss,episode,done=self.train_() # Call the train_ method and get loss value, episode list, and done flag + if self.trial_count!=None: # If trial count value is not None (need to calculate average reward) + if len(self.reward_list)>=self.trial_count: # If the length of the reward list is greater than or equal to the trial count value + avg_reward=statistics.mean(self.reward_list[-self.trial_count:]) # Calculate the average reward by using the mean function on the last trial count elements of the reward list + if self.criterion!=None and avg_reward>=self.criterion: # If criterion value is not None and average reward is greater than or equal to criterion value (achieved the goal) + t2=time.time() # Record the end time of the episode + self.total_time+=(t2-t1) # Add the episode time to the total time + time_=self.total_time-int(self.total_time) # Get the decimal part of the total time + if time_<0.5: # If the decimal part is less than 0.5 + self.total_time=int(self.total_time) # Round down the total time + else: # If the decimal part is greater than or equal to 0.5 + self.total_time=int(self.total_time)+1 # Round up the total time + print('episode:{0}'.format(self.total_episode)) # Print the current episode number + print('last loss:{0:.6f}'.format(loss)) # Print the last loss value with six decimal places + print('average reward:{0}'.format(avg_reward)) # Print the average reward value + print() + print('time:{0}s'.format(self.total_time)) # Print the total time in seconds + return # Return nothing and exit the method + self.loss=loss # Set loss attribute to be equal to loss value + self.loss_list.append(loss) # Append loss value to loss list + self.total_episode+=1 # Increase total episode by one + if episode_count%10!=0: # If episode count is not divisible by 10 + p=episode_count-episode_count%self.p # Calculate p by subtracting the remainder of dividing episode count by p from episode count + p=int(p/self.p) # Divide p by p and round down + s=episode_count-episode_count%self.s # Calculate s by subtracting the remainder of dividing episode count by s from episode count + s=int(s/self.s) # Divide s by s and round down + else: # If episode count is divisible by 10 + p=episode_count/(self.p+1) # Divide episode count by p plus one and round down + p=int(p) # Convert p to integer + s=episode_count/(self.s+1) # Divide episode count by s plus one and round down + s=int(s) # Convert s to integer + if p==0: # If p is zero (should not happen) + p=1 # Set it to be one (print every episode) + if s==0: # If s is zero (should not happen) + s=1 # Set it to be one (save every episode) + if i%p==0: # If i (current episode number) is divisible by p (print frequency) + if len(self.state_pool)>=self.batch: # If the length of the state pool is greater than or equal to the batch size (enough data for training) + print('episode:{0} loss:{1:.6f}'.format(i+1,loss)) # Print the current episode number and loss value with six decimal places + if avg_reward!=None: # If average reward value is not None (calculated average reward) + print('episode:{0} average reward:{1}'.format(i+1,avg_reward)) # Print the current episode number and average reward value + else: # If average reward value is None (did not calculate average reward) + print('episode:{0} reward:{1}'.format(i+1,self.reward)) # Print the current episode number and total reward value + print() + if save!=None and i%s==0: # If save value is not None (need to save data) and i (current episode number) is divisible by s (save frequency) + self.save(self.total_episode,one) # Call the save method with total episode and one flag as inputs + if self.save_episode==True: # If save episode flag is True (need to save episodes) + if done: # If done flag is True (episode ended) + episode.append('done') # Append 'done' to episode list + self.episode_set.append(episode) # Append episode list to episode set list + if self.max_episode_count!=None and len(self.episode_set)>=self.max_episode_count: # If max episode count value is not None (there is a limit on episodes to save) and the length of episode set list is greater than or equal to max episode count value + self.save_episode=False # Set save episode flag to False (stop saving episodes) + try: # Try to execute the following code block + self.nn.ec.assign_add(1) # Add one to the model's episode counter using assign_add method + except Exception: # If an exception occurs + pass # Do nothing and ignore the exception + t2=time.time() # Record the end time of the episode + self.time+=(t2-t1) # Add the episode time to the time attribute + else: # If episode count value is None (no limit on episodes) + i=0 # Initialize i to zero + while True: # Loop indefinitely until break statement is executed + t1=time.time() # Record the start time of the episode + loss,episode,done=self.train_() # Call the train_ method and get loss value, episode list, and done flag + if self.trial_count!=None: # If trial count value is not None (need to calculate average reward) + if len(self.reward_list)==self.trial_count: # If the length of the reward list is equal to the trial count value + avg_reward=statistics.mean(self.reward_list[-self.trial_count:]) # Calculate the average reward by using the mean function on the last trial count elements of the reward list + if self.criterion!=None and avg_reward>=self.criterion: # If criterion value is not None and average reward is greater than or equal to criterion value (achieved the goal) + t2=time.time() # Record the end time of the episode + self.total_time+=(t2-t1) # Add the episode time to the total time + time_=self.total_time-int(self.total_time) # Get the decimal part of the total time + if time_<0.5: # If the decimal part is less than 0.5 + self.total_time=int(self.total_time) # Round down the total time + else: # If the decimal part is greater than or equal to 0.5 + self.total_time=int(self.total_time)+1 # Round up the total time + print('episode:{0}'.format(self.total_episode)) # Print the current episode number + print('last loss:{0:.6f}'.format(loss)) # Print the last loss value with six decimal places + print('average reward:{0}'.format(avg_reward)) # Print the average reward value + print() + print('time:{0}s'.format(self.total_time)) # Print the total time in seconds + return # Return nothing and exit the method + self.loss=loss # Set loss attribute to be equal to loss value + self.loss_list.append(loss) # Append loss value to loss list + i+=1 # Increase i by one + self.total_episode+=1 # Increase total episode by one + if episode_count%10!=0: # If episode count is not divisible by 10 + p=episode_count-episode_count%self.p # Calculate p by subtracting the remainder of dividing episode count by p from episode count + p=int(p/self.p) # Divide p by p and round down + s=episode_count-episode_count%self.s # Calculate s by subtracting the remainder of dividing episode count by s from episode count + s=int(s/self.s) # Divide s by s and round down + else: # If episode count is divisible by 10 + p=episode_count/(self.p+1) # Divide episode count by p plus one and round down + p=int(p) # Convert p to integer + s=episode_count/(self.s+1) # Divide episode count by s plus one and round down + s=int(s) # Convert s to integer + if i%p==0: # If i (current episode number) is divisible by p (print frequency) + if len(self.state_pool)>=self.batch: # If the length of the state pool is greater than or equal to the batch size (enough data for training) + print('episode:{0} loss:{1:.6f}'.format(i+1,loss)) # Print the current episode number and loss value with six decimal places + if avg_reward!=None: # If average reward value is not None (calculated average reward) + print('episode:{0} average reward:{1}'.format(i+1,avg_reward)) # Print the current episode number and average reward value + else: # If average reward value is None (did not calculate average reward) + print('episode:{0} reward:{1}'.format(i+1,self.reward)) # Print the current episode number and total reward value + print() + if save!=None and i%s==0: # If save value is not None (need to save data) and i (current episode number) is divisible by s (save frequency) + self.save(self.total_episode,one) # Call the save method with total episode and one flag as inputs + if self.save_episode==True: # If save episode flag is True (need to save episodes) + if done: # If done flag is True (episode ended) + episode.append('done') # Append 'done' to episode list + self.episode_set.append(episode) # Append episode list to episode set list + if self.max_episode_count!=None and len(self.episode_set)>=self.max_episode_count: # If max episode count value is not None (there is a limit on episodes to save) and the length of episode set list is greater than or equal to max episode count value + self.save_episode=False # Set save episode flag to False (stop saving episodes) + try: # Try to execute the following code block + self.nn.ec.assign_add(1) # Add one to the model's episode counter using assign_add method + except Exception: # If an exception occurs + self.nn.ec+=1 # Use normal addition instead + t2=time.time() # Record the end time of the episode + self.time+=(t2-t1) # Add the episode time to the time attribute + time_=self.time-int(self.time) # Get the decimal part of the time attribute + if time_<0.5: # If the decimal part is less than 0.5 + self.total_time=int(self.time) # Round down the time attribute + else: # If the decimal part is greater than or equal to 0.5 + self.total_time=int(self.time)+1 # Round up the time attribute + self.total_time+=self.time # Add the time attribute to the total time attribute + print('last loss:{0:.6f}'.format(loss)) # Print the last loss value with six decimal places + print('last reward:{0}'.format(self.reward)) # Print the last reward value + print() + print('time:{0}s'.format(self.time)) # Print the time attribute in seconds + return # Return nothing - # This is a function for training the model using data from episodes or pools - def train(self,episode_count,save=None,one=True,p=None,s=None): - avg_reward=None # This is to initialize the average reward value to None - if p==None: # This is a condition to check if the p parameter is None, which indicates that it needs to be assigned a default value - self.p=9 # This is to assign 9 to the p attribute, which stands for the frequency of printing the loss and reward values - else: # This means that the p parameter is not None, which indicates that it has a given value - self.p=p-1 # This is to assign p-1 to the p attribute, which stands for the frequency of printing the loss and reward values - if s==None: # This is a condition to check if the s parameter is None, which indicates that it needs to be assigned a default value - self.s=1 # This is to assign 1 to the s attribute, which stands for the frequency of saving the model parameters - self.file_list=None # This is to assign None to the file_list attribute, which stands for a list of file names for saving the model parameters - else: # This means that the s parameter is not None, which indicates that it has a given value - self.s=s-1 # This is to assign s-1 to the s attribute, which stands for the frequency of saving the model parameters - self.file_list=[] # This is to initialize an empty list for the file_list attribute, which stands for a list of file names for saving the model parameters - if episode_count!=None: # This is a condition to check if the episode_count parameter is not None, which indicates that it has a given value - for i in range(episode_count): # This is a loop for running a given number of episodes - t1=time.time() # This is to record the start time of an episode - loss,episode,done=self.train_() # This is to call the train_ method for training the model using data from an episode and return the loss value, episode and done flag - if self.trial_count!=None: # This is a condition to check if the trial_count attribute is not None, which indicates that it has a defined value - if len(self.reward_list)>=self.trial_count: # This is a condition to check if the length of the reward_list attribute, which stands for a list of rewards per episode, is greater than or equal to the trial_count attribute, which stands for the number of episodes for calculating the average reward - avg_reward=statistics.mean(self.reward_list[-self.trial_count:]) # This is to calculate the average reward value by taking the mean of the last trial_count elements of the reward_list attribute - if self.criterion!=None and avg_reward>=self.criterion: # This is a condition to check if the criterion attribute is not None and the average reward value is greater than or equal to the criterion attribute, which indicates that the model has reached a desired performance level - t2=time.time() # This is to record the end time of an episode - self.total_time+=(t2-t1) # This is to update the total_time attribute, which stands for the total time spent on training, by adding (t2-t1), which stands for the time spent on an episode - time_=self.total_time-int(self.total_time) # This is to calculate the decimal part of the total_time attribute - if time_<0.5: # This is a condition to check if the decimal part of the total_time attribute is less than 0.5, which indicates that it needs rounding down - self.total_time=int(self.total_time) # This is to round down the total_time attribute by converting it into an integer - else: # This means that the decimal part of the total_time attribute is greater than or equal to 0.5, which indicates that it needs rounding up - self.total_time=int(self.total_time)+1 # This is to round up - print('episode:{0}'.format(self.total_episode)) # This is to print the episode number - print('last loss:{0:.6f}'.format(loss)) # This is to print the last loss value with six decimal places - print('average reward:{0}'.format(avg_reward)) # This is to print the average reward value - print() # This is to print an empty line for spacing - print('time:{0}s'.format(self.total_time)) # This is to print the total time spent on training in seconds - return # This is to return from the function - self.loss=loss # This is to assign the loss value to the loss attribute - self.loss_list.append(loss) # This is to append the loss value to the loss_list attribute, which stands for a list of losses per episode - self.total_episode+=1 # This is to increase the total_episode attribute, which stands for the total number of episodes, by one - if episode_count%10!=0: # This is a condition to check if the episode_count parameter is not divisible by 10, which indicates that it needs some adjustment for calculating p and s - p=episode_count-episode_count%self.p # This is to subtract the remainder of dividing the episode_count parameter by the p attribute from the episode_count parameter - p=int(p/self.p) # This is to divide p by the p attribute and convert it into an integer - s=episode_count-episode_count%self.s # This is to subtract the remainder of dividing the episode_count parameter by the s attribute from the episode_count parameter - s=int(s/self.s) # This is to divide s by the s attribute and convert it into an integer - else: # This means that the episode_count parameter is divisible by 10, which indicates that it does not need any adjustment for calculating p and s - p=episode_count/(self.p+1) # This is to divide the episode_count parameter by (the p attribute plus one) - p=int(p) # This is to convert p into an integer - s=episode_count/(self.s+1) # This is to divide the episode_count parameter by (the s attribute plus one) - s=int(s) # This is to convert s into an integer - if p==0: # This is a condition to check if p is zero, which indicates that it needs to be assigned a minimum value of one - p=1 # This is to assign one to p - if s==0: # This is a condition to check if s is zero, which indicates that it needs to be assigned a minimum value of one - s=1 # This is to assign one to s - if i%p==0: # This is a condition to check if i, which stands for the episode index, is divisible by p, which indicates that it is time to print the loss and reward values - if len(self.state_pool)>=self.batch: # This is a condition to check if the length of the state_pool attribute, which stands for a pool of states, is greater than or equal to the batch attribute, which stands for a batch size for training - print('episode:{0} loss:{1:.6f}'.format(i+1,loss)) # This is to print the episode number and the loss value with six decimal places - if avg_reward!=None: # This is a condition to check if the average reward value is not None, which indicates that it has been calculated - print('episode:{0} average reward:{1}'.format(i+1,avg_reward)) # This is to print the episode number and the average reward value - else: # This means that the average reward value is None, which indicates that it has not been calculated yet - print('episode:{0} reward:{1}'.format(i+1,self.reward)) # This is to print the episode number and the reward value - print() # This is to print an empty line for spacing - if save!=None and i%s==0: # This is a condition to check if the save parameter is not None and i, which stands for the episode index, is divisible by s, which indicates that it is time to save the model parameters - self.save(self.total_episode,one) # This is to call the save method for saving the model parameters using the total_episode attribute as an identifier and one as a flag for indicating whether to overwrite or append files - if self.save_episode==True: # This is a condition to check if the save_episode attribute is True, which indicates that it needs to save an episode - if done: # This is a condition to check if done flag - episode.append('done') - self.episode_set.append(episode) # This is to append the episode list to the episode_set attribute, which stands for a set of episodes - if self.max_episode_count!=None and len(self.episode_set)>=self.max_episode_count: # This is a condition to check if the max_episode_count attribute is not None and the length of the episode_set attribute is greater than or equal to the max_episode_count attribute, which indicates that it has reached the maximum number of episodes to save - self.save_episode=False # This is to assign False to the save_episode attribute, which indicates that it does not need to save any more episodes - try: # This is a try block for handling any possible errors - try: # This is a nested try block for handling any possible errors - self.nn.ec.assign_add(1) # This is to call the assign_add method of the ec attribute of the model, which stands for episode counter, to increase it by one - except Exception: # This is an except block for catching any exceptions raised by the nested try block - self.nn.ec+=1 # This is to increase the ec attribute of the model by one using the += operator - except Exception: # This is an except block for catching any exceptions raised by the outer try block - pass # This is a pass statement for doing nothing and continuing the execution - t2=time.time() # This is to record the end time of an episode - self.time+=(t2-t1) # This is to update the time attribute, which stands for the time spent on an episode, by adding (t2-t1), which stands for the time difference between t2 and t1 - else: # This is an else clause for the condition of the episode_count parameter, which indicates that it does not have a given value - i=0 # This is to initialize the episode index to zero - while True: # This is an infinite loop for running episodes until the model reaches the desired performance level - t1=time.time() # This is to record the start time of an episode - loss,episode,done=self.train_() # This is to call the train_ method for training the model using data from an episode and return the loss value, episode and done flag - if self.trial_count!=None: # This is a condition to check if the trial_count attribute is not None, which indicates that it has a defined value - if len(self.reward_list)==self.trial_count: # This is a condition to check if the length of the reward_list attribute, which stands for a list of rewards per episode, is equal to the trial_count attribute, which stands for the number of episodes for calculating the average reward - avg_reward=statistics.mean(self.reward_list[-self.trial_count:]) # This is to calculate the average reward value by taking the mean of the last trial_count elements of the reward_list attribute - if self.criterion!=None and avg_reward>=self.criterion: # This is a condition to check if the criterion attribute is not None and the average reward value is greater than or equal to the criterion attribute, which indicates that the model has reached a desired performance level - t2=time.time() # This is to record the end time of an episode - self.total_time+=(t2-t1) # This is to update the total_time attribute, which stands for the total time spent on training, by adding (t2-t1), which stands for the time spent on an episode - time_=self.total_time-int(self.total_time) # This is to calculate the decimal part of the total_time attribute - if time_<0.5: # This is a condition to check if the decimal part of the total_time attribute is less than 0.5, which indicates that it needs rounding down - self.total_time=int(self.total_time) # This is to round down the total_time attribute by converting it into an integer - else: # This means that the decimal part of the total_time attribute is greater than or equal to 0.5, which indicates that it needs rounding up - self.total_time=int(self.total_time)+1 # This is to round up - print('episode:{0}'.format(self.total_episode)) # This is to print the episode number - print('last loss:{0:.6f}'.format(loss)) # This is to print the last loss value with six decimal places - print('average reward:{0}'.format(avg_reward)) # This is to print the average reward value - print() # This is to print an empty line for spacing - print('time:{0}s'.format(self.total_time)) # This is to print the total time spent on training in seconds - return # This is to return from the function - self.loss=loss # This is to assign the loss value to the loss attribute - self.loss_list.append(loss) # This is to append the loss value to the loss_list attribute, which stands for a list of losses per episode - self.total_episode+=1 # This is to increase the total_episode attribute, which stands for the total number of episodes, by one - if episode_count%10!=0: # This is a condition to check if the episode_count parameter is not divisible by 10, which indicates that it needs some adjustment for calculating p and s - p=episode_count-episode_count%self.p # This is to subtract the remainder of dividing the episode_count parameter by the p attribute from the episode_count parameter - p=int(p/self.p) # This is to divide p by the p attribute and convert it into an integer - s=episode_count-episode_count%self.s # This is to subtract the remainder of dividing the episode_count parameter by the s attribute from the episode_count parameter - s=int(s/self.s) # This is to divide s by the s attribute and convert it into an integer - else: # This means that the episode_count parameter is divisible by 10, which indicates that it does not need any adjustment for calculating p and s - p=episode_count/(self.p+1) # This is to divide the episode_count parameter by (the p attribute plus one) - p=int(p) # This is to convert p into an integer - s=episode_count/(self.s+1) # This is to divide the episode_count parameter by (the s attribute plus one) - s=int(s) # This is to convert s into an integer - if p==0: # This is a condition to check if p is zero, which indicates that it needs to be assigned a minimum value of one - p=1 # This is to assign one to p - if s==0: # This is a condition to check if s is zero, which indicates that it needs to be assigned a minimum value of one - s=1 # This is to assign one to s - if i%p==0: # This is a condition to check if i, which stands for the episode index, is divisible by p, which indicates that it is time to print the loss and reward values - if len(self.state_pool)>=self.batch: # This is a condition to check if the length of the state_pool attribute, which stands for a pool of states, is greater than or equal to the batch attribute, which stands for a batch size for training - print('episode:{0} loss:{1:.6f}'.format(i+1,loss)) # This is to print the episode number and the loss value with six decimal places - if avg_reward!=None: # This is a condition to check if the average reward value is not None, which indicates that it has been calculated - print('episode:{0} average reward:{1}'.format(i+1,avg_reward)) # This is to print the episode number and the average reward value - else: # This means that the average reward value is None, which indicates that it has not been calculated yet - print('episode:{0} reward:{1}'.format(i+1,self.reward)) # This is to print the episode number and the reward value - print() # This is to print an empty line for spacing - if save!=None and i%s==0: # This is a condition to check if the save parameter is not None and i, which stands for the episode index, is divisible by s, which indicates that it is time to save the model parameters - self.save(self.total_episode,one) # This is to call the save method for saving the model parameters using the total_episode attribute as an identifier and one as a flag for indicating whether to overwrite or append files - if self.save_episode==True: # This is a condition to check if the save_episode attribute is True, which indicates that it needs to save an episode - if done: # This is a condition to check if done flag - episode.append('done') - self.episode_set.append(episode) # This is to append the episode list to the episode_set attribute, which stands for a set of episodes - if self.max_episode_count!=None and len(self.episode_set)>=self.max_episode_count: # This is a condition to check if the max_episode_count attribute is not None and the length of the episode_set attribute is greater than or equal to the max_episode_count attribute, which indicates that it has reached the maximum number of episodes to save - self.save_episode=False # This is to assign False to the save_episode attribute, which indicates that it does not need to save any more episodes - try: # This is a try block for handling any possible errors - try: # This is a nested try block for handling any possible errors - self.nn.ec.assign_add(1) # This is to call the assign_add method of the ec attribute of the model, which stands for episode counter, to increase it by one - except Exception: # This is an except block for catching any exceptions raised by the nested try block - self.nn.ec+=1 # This is to increase the ec attribute of the model by one using the += operator - except Exception: # This is an except block for catching any exceptions raised by the outer try block - pass # This is a pass statement for doing nothing and continuing the execution - t2=time.time() # This is to record the end time of an episode - self.time+=(t2-t1) # This is to update the time attribute, which stands for the time spent on an episode, by adding (t2-t1), which stands for the time difference between t2 and t1 - time_=self.time-int(self.time) - time_=self.time-int(self.time) # This is to calculate the decimal part of the time attribute, which stands for the time spent on an episode - if time_<0.5: # This is a condition to check if the decimal part of the time attribute is less than 0.5, which indicates that it needs rounding down - self.total_time=int(self.time) # This is to round down the time attribute by converting it into an integer and assign it to the total_time attribute, which stands for the total time spent on training - else: # This means that the decimal part of the time attribute is greater than or equal to 0.5, which indicates that it needs rounding up - self.total_time=int(self.time)+1 # This is to round up the time attribute by converting it into an integer and adding one and assign it to the total_time attribute - self.total_time+=self.time # This is to update the total_time attribute by adding the time attribute to it - print('last loss:{0:.6f}'.format(loss)) # This is to print the last loss value with six decimal places - print('last reward:{0}'.format(self.reward)) # This is to print the last reward value - print() # This is to print an empty line for spacing - print('time:{0}s'.format(self.time)) # This is to print the time spent on an episode in seconds - return - - # This is a function for training the model online using data from the environment - def train_online(self): - while True: # This is an infinite loop for running online training until the model stops or suspends - if hasattr(self.nn,'save'): # This is a condition to check if the model has a save attribute, which stands for a save function - self.nn.save(self.save) # This is to call the model's save method for saving the model parameters using the save attribute, which stands for a file name - if hasattr(self.nn,'stop_flag'): # This is a condition to check if the model has a stop_flag attribute, which stands for a flag for stopping the training - if self.nn.stop_flag==True: # This is a condition to check if the stop_flag attribute is True, which indicates that the training needs to stop - return # This is to return from the function - if hasattr(self.nn,'stop_func'): # This is a condition to check if the model has a stop_func attribute, which stands for a stop function - if self.nn.stop_func(): # This is to call the model's stop_func method and check if it returns True, which indicates that the training needs to stop - return # This is to return from the function - if hasattr(self.nn,'suspend_func'): # This is a condition to check if the model has a suspend_func attribute, which stands for a suspend function - self.nn.suspend_func() # This is to call the model's suspend_func method for suspending the training if needed - data=self.nn.online() # This is to call the model's online method for getting data from the environment and assign it to data - if data=='stop': # This is a condition to check if data is 'stop', which indicates that the training needs to stop - return # This is to return from the function - elif data=='suspend': # This is a condition to check if data is 'suspend', which indicates that the training needs to suspend - self.nn.suspend_func() # This is to call the model's suspend_func method for suspending the training - loss=self.opt_ol(data[0],data[1],data[2],data[3],data[4]) # This is to call the opt_ol method for optimizing the model using data and return the loss value. The data consists of five elements: state, action, next state, reward and done flag. - loss=loss.numpy() # This is to convert the loss value into a numpy array - self.nn.train_loss_list.append(loss) # This is to append the loss value to the train_loss_list attribute of the model, which stands for a list of losses per step - if len(self.nn.train_acc_list)==self.nn.max_length: # This is a condition to check if the length of the train_acc_list attribute of the model, which stands for a list of accuracies per step, is equal to the max_length attribute of the model, which stands for the maximum length of lists - del self.nn.train_acc_list[0] # This is to delete or pop out the first element of the train_acc_list attribute of the model, which stands for removing the oldest accuracy value - if hasattr(self.nn,'counter'): # This is a condition to check if the model has a counter attribute, which stands for a counter for steps or epochs - self.nn.counter+=1 # This is to increase the counter attribute - return \ No newline at end of file + def train_online(self): # This is a method for training the network online (without using a pool) + while True: # Loop indefinitely until break statement is executed + if hasattr(self.nn,'save'): # If the model has a save attribute + self.nn.save(self.save) # Call the model's save method with save attribute as input + if hasattr(self.nn,'stop_flag'): # If the model has a stop flag attribute + if self.nn.stop_flag==True: # If the stop flag is True (need to stop training) + return # Return nothing and exit the method + if hasattr(self.nn,'stop_func'): # If the model has a stop function attribute + if self.nn.stop_func(): # If the stop function returns True (need to stop training) + return # Return nothing and exit the method + if hasattr(self.nn,'suspend_func'): # If the model has a suspend function attribute + self.nn.suspend_func() # Call the suspend function to check if training needs to be suspended + data=self.nn.online() # Get online data by using model's online method + if data=='stop': # If data is 'stop' (need to stop training) + return # Return nothing and exit the method + elif data=='suspend': # If data is 'suspend' (need to suspend training) + self.nn.suspend_func() # Call the suspend function to suspend training + loss=self.opt_ol(data[0],data[1],data[2],data[3],data[4]) # Get loss value by using opt_ol method with data as inputs + loss=loss.numpy() # Convert loss value to numpy array + self.nn.train_loss_list.append(loss) # Append loss value to model's train loss list + if len(self.nn.train_acc_list)==self.nn.max_length: # If the length of model's train accuracy list is equal to model's max length value + del self.nn.train_acc_list[0] # Delete the first element of model's train accuracy list + if hasattr(self.nn,'counter'): # If the model has a counter attribute + self.nn.counter+=1 # Increase the counter by one + return # Return nothing From 3c5f3ec38abcf113c9b89e62fa7ac73684ea919b Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Fri, 28 Jul 2023 13:33:46 +0800 Subject: [PATCH 129/337] Update README.md --- README.md | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index da135200..c31e3cae 100644 --- a/README.md +++ b/README.md @@ -51,13 +51,11 @@ kernel.train(32,5) #train the network with batch size 32 and epoch 5 import Note.RL.kernel as k #import kernel module import DQN as d #import deep Q-network module dqn=d.DQN(4,128,2) #create neural network object with 4 inputs, 128 hidden units and 2 outputs -kernel=k.kernel(dqn,5) #create kernel object with the network and 5 threads to train +kernel=k.kernel(dqn) #create kernel object with the network kernel.stop=True #set the flag to stop training when a condition is met kernel.action_count=2 #set the number of actions to 2 kernel.set_up(epsilon=0.01,pool_size=10000,batch=64,update_step=10,trial_count=10,criterion=200) #set up the hyperparameters for training and the condition to stop training when the average reward of 10 trials is greater than 200 kernel.train(500) #train the network for 500 episodes -kernel.visualize_train() -kernel.visualize_reward() ``` ## Training with test data @@ -375,6 +373,25 @@ for p in range(5): #loop over the processes Process(target=kernel.train,args=(p,100,lock,pool_lock)).start() #start each process with the train function and pass the process id, the number of episodes, the locks and the pool locks as arguments ``` +### Stop training and saving when condition is met: +```python +import Note.RL.parallel.kernel as k #import kernel module +import DQN as d #import deep Q-network module +from multiprocessing import Process,Lock,Manager #import multiprocessing tools +dqn=d.DQN(4,128,2) #create neural network object with 4 inputs, 128 hidden units and 2 outputs +kernel=k.kernel(dqn,5) #create kernel object with the network and 5 processes to train +manager=Manager() #create manager object to share data among processes +kernel.init(manager) #initialize shared data with the manager +kernel.action_count=2 #set the number of actions to 2 +kernel.stop=True #set the flag to stop training when a condition is met +kernel.set_up(epsilon=0.01,pool_size=10000,batch=64,update_step=10,trial_count=10,criterion=200) #set up the hyperparameters for training +kernel.PO=3 #use PO3 algorithm for parallel optimization +pool_lock=[Lock(),Lock(),Lock(),Lock(),Lock()] #create a list of locks for each process's replay pool +lock=[Lock(),Lock(),Lock()] #create three locks for synchronization +for p in range(5): #loop over the processes + Process(target=kernel.train,args=(p,100,lock,pool_lock)).start() #start each process with the train function and pass the process id, the number of episodes, the locks and the pool locks as arguments +``` + # Parallel test: **You can get neural network example from the link below, and then you can import neural network and train with kernel, example code are below.** From 5c61a1b235641a9fa5f3465cbe3a12da5ef756a1 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Fri, 28 Jul 2023 13:34:55 +0800 Subject: [PATCH 130/337] Update and rename Note 7.0 documentation/RL/kernel/nspn/kernel.py to Note 7.0 documentation/RL/kernel/kernel.py --- Note 7.0 documentation/RL/kernel/{nspn => }/kernel.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename Note 7.0 documentation/RL/kernel/{nspn => }/kernel.py (100%) diff --git a/Note 7.0 documentation/RL/kernel/nspn/kernel.py b/Note 7.0 documentation/RL/kernel/kernel.py similarity index 100% rename from Note 7.0 documentation/RL/kernel/nspn/kernel.py rename to Note 7.0 documentation/RL/kernel/kernel.py From 512cde92178ecca75a534e7b897fd336be2b2d97 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Fri, 28 Jul 2023 14:09:26 +0800 Subject: [PATCH 131/337] Update kernel_pytorch.py --- Note 7.0 documentation/DL/kernel/parallel/kernel_pytorch.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Note 7.0 documentation/DL/kernel/parallel/kernel_pytorch.py b/Note 7.0 documentation/DL/kernel/parallel/kernel_pytorch.py index 45939718..6958343c 100644 --- a/Note 7.0 documentation/DL/kernel/parallel/kernel_pytorch.py +++ b/Note 7.0 documentation/DL/kernel/parallel/kernel_pytorch.py @@ -24,7 +24,6 @@ def __init__(self,nn=None): # define the constructor method of the class self.end_test_loss=None # initialize the end_test_loss attribute to None self.end_test_acc=None # initialize the end_test_acc attribute to None self.acc_flag='%' # initialize the acc_flag attribute to '%' - self.p=None # initialize the p attribute to None self.s=None # initialize the s attribute to None self.saving_one=True # initialize the saving_one attribute to True self.filename='save.dat' # initialize the filename attribute to 'save.dat' @@ -286,7 +285,7 @@ def train7(self,train_loader,p,test_batch,lock): else: # otherwise self.test_loss.value=self.test(test_batch) # call the test method with the test_batch argument, and assign the return value to the value of the test_loss attribute self.test_loss_list.append(self.test_loss.value) # append the value of the test_loss attribute to the test_loss_list attribute - self.print_save() # call the print_save method to print and save some information + self.save_() # call the save_ method to save self.epoch_counter.value+=1 # increment the value of the epoch_counter attribute by 1 if hasattr(self.nn,'ec'): # check if the nn attribute has an attribute named ec ec=self.nn.ec[0] # assign the first element of the ec attribute of the nn attribute to the ec variable From 1054d5306394d999d84530bee94410d24ad1b9eb Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Fri, 28 Jul 2023 14:09:47 +0800 Subject: [PATCH 132/337] Update kernel.py --- Note 7.0 documentation/DL/kernel/parallel/kernel.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Note 7.0 documentation/DL/kernel/parallel/kernel.py b/Note 7.0 documentation/DL/kernel/parallel/kernel.py index f3e68f45..aa350ccd 100644 --- a/Note 7.0 documentation/DL/kernel/parallel/kernel.py +++ b/Note 7.0 documentation/DL/kernel/parallel/kernel.py @@ -31,8 +31,7 @@ def __init__(self,nn=None): self.end_test_loss=None # the end condition for testing loss self.end_test_acc=None # the end condition for testing accuracy self.acc_flag='%' # a flag to indicate whether to use percentage or decimal for accuracy display - self.p=None # the process index for online mode - self.s=None # a signal object for online mode + self.s=None # initialize the s attribute to None self.saving_one=True # a flag to indicate whether to save only one copy of model parameters or multiple copies with different names self.filename='save.dat' # the default filename for saving model parameters self.test_flag=False # a flag to indicate whether to use testing data or not @@ -391,7 +390,7 @@ def train7(self,train_ds,p,test_batch,lock,g_lock): else: self.test_loss.value=self.test(self.test_data,self.test_labels,test_batch) # get the testing loss value with testing data, labels and batch size as inputs self.test_loss_list.append(self.test_loss.value) # append the testing loss value to the testing loss list - self.print_save() # print and save the results for this epoch + self.save_() # save the results for this epoch self.epoch_counter.value+=1 # increment the epoch counter by 1 if hasattr(self.nn,'ec'): ec=self.nn.ec[0] # get the epoch counter in the neural network model From 3a0619839f57153dc4d9294a2d3b939b1cec8e0a Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Fri, 28 Jul 2023 14:10:38 +0800 Subject: [PATCH 133/337] Update kernel.py --- Note 7.0 documentation/DL/kernel/parallel/kernel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Note 7.0 documentation/DL/kernel/parallel/kernel.py b/Note 7.0 documentation/DL/kernel/parallel/kernel.py index aa350ccd..9e81b4e2 100644 --- a/Note 7.0 documentation/DL/kernel/parallel/kernel.py +++ b/Note 7.0 documentation/DL/kernel/parallel/kernel.py @@ -390,7 +390,7 @@ def train7(self,train_ds,p,test_batch,lock,g_lock): else: self.test_loss.value=self.test(self.test_data,self.test_labels,test_batch) # get the testing loss value with testing data, labels and batch size as inputs self.test_loss_list.append(self.test_loss.value) # append the testing loss value to the testing loss list - self.save_() # save the results for this epoch + self.save_() # call the save_ method to save self.epoch_counter.value+=1 # increment the epoch counter by 1 if hasattr(self.nn,'ec'): ec=self.nn.ec[0] # get the epoch counter in the neural network model From 522dbd456da9486f5c3ad8c32a0adbeb23d902b5 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Fri, 28 Jul 2023 22:54:47 +0800 Subject: [PATCH 134/337] Update nn_attenuate.py --- .../tensorflow/parallel/nn_attenuate.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn_attenuate.py b/Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn_attenuate.py index 24d2836b..73e09865 100644 --- a/Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn_attenuate.py +++ b/Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn_attenuate.py @@ -1,4 +1,5 @@ import tensorflow as tf +from tensorflow.python.util import nest import Note.nn.layer.dense as d from Note.nn.parallel.optimizer import Momentum from Note.nn.layer.flatten import flatten @@ -37,13 +38,15 @@ def loss(self,output,labels): def attenuate(self,gradient,oc,p): #gradient attenuation function,kernel uses it to calculate attenuation coefficient. # Apply an exponential decay to the gradient based on the optimization counter - ac=0.9**oc[p] #ac:attenuation coefficient - for i in range(len(gradient)): #oc:optimization counter - gradient[i]=ac*gradient[i] #p:process number + ac=0.9**oc[0][p] #ac:attenuation coefficient + gradient_flat=nest.flatten(gradient) + for i in range(len(gradient_flat)): #oc:optimization counter + gradient_flat[i]=ac*gradient_flat[i] #p:process number + gradient=nest.pack_sequence_as(gradient,gradient_flat) return gradient def opt(self,gradient): # Perform optimization on the parameters using the gradient param=self.optimizer.opt(gradient,self.param) - return param \ No newline at end of file + return param From ae17e9412252f90ce8e97df9b1c72988eb772704 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Fri, 28 Jul 2023 23:01:43 +0800 Subject: [PATCH 135/337] Update nn_attenuate.py --- .../tensorflow/parallel/nn_attenuate.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn_attenuate.py b/Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn_attenuate.py index 73e09865..3f09c8b8 100644 --- a/Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn_attenuate.py +++ b/Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn_attenuate.py @@ -3,6 +3,7 @@ import Note.nn.layer.dense as d from Note.nn.parallel.optimizer import Momentum from Note.nn.layer.flatten import flatten +from Note.nn.accuracy import sparse_categorical_accuracy # Define a neural network class with gradient attenuation class nn: @@ -25,28 +26,32 @@ def build(self): def fp(self,data): # Perform forward propagation on the input data - data=flatten(data) - output1=self.layer1.output(data) - output2=self.layer2.output(output1) + data=flatten(data) # Flatten the data to a one-dimensional vector + output1=self.layer1.output(data) # Apply the first layer to the data and get the output + output2=self.layer2.output(output1) # Apply the second layer to the output of the first layer and get the final output return output2 def loss(self,output,labels): # Compute the loss between the output and the labels - return self.loss_object(labels,output) + return self.loss_object(labels,output) # Use the loss function to calculate the loss + + + def accuracy(self,output,labels): #accuracy function,kernel uses it to calculate accuracy. + return sparse_categorical_accuracy(labels,output) def attenuate(self,gradient,oc,p): #gradient attenuation function,kernel uses it to calculate attenuation coefficient. # Apply an exponential decay to the gradient based on the optimization counter ac=0.9**oc[0][p] #ac:attenuation coefficient - gradient_flat=nest.flatten(gradient) + gradient_flat=nest.flatten(gradient) # Flatten the gradient to a one-dimensional vector for i in range(len(gradient_flat)): #oc:optimization counter gradient_flat[i]=ac*gradient_flat[i] #p:process number - gradient=nest.pack_sequence_as(gradient,gradient_flat) + gradient=nest.pack_sequence_as(gradient,gradient_flat) # Restore the gradient to its original shape return gradient def opt(self,gradient): # Perform optimization on the parameters using the gradient - param=self.optimizer.opt(gradient,self.param) + param=self.optimizer.opt(gradient,self.param) # Use the optimizer to update the parameters return param From 16be41db331f2dd321a49262ce2dacfc07f5a160 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Fri, 28 Jul 2023 23:15:36 +0800 Subject: [PATCH 136/337] Update README.md --- README.md | 46 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index c31e3cae..12ccd2f3 100644 --- a/README.md +++ b/README.md @@ -113,16 +113,16 @@ x_train,x_test =x_train/255.0,x_test/255.0 #normalize data nn=n.nn() #create neural network object nn.build() #build the network structure kernel=k.kernel(nn) #create kernel object with the network -kernel.process=7 #set the number of processes to train +kernel.process=3 #set the number of processes to train kernel.data_segment_flag=True #set the flag to segment data for each process -kernel.epoch=6 #set the number of epochs to train +kernel.epoch=5 #set the number of epochs to train kernel.batch=32 #set the batch size kernel.PO=1 #use PO1 algorithm for parallel optimization kernel.data(x_train,y_train) #input train data to the kernel manager=Manager() #create manager object to share data among processes kernel.init(manager) #initialize shared data with the manager lock=[Lock(),Lock()] #create two locks for synchronization -for p in range(7): #loop over the processes +for p in range(3): #loop over the processes Process(target=kernel.train,args=(p,lock)).start() #start each process with the train function and pass the process id and locks as arguments kernel.update_nn_param() #update the network parameters after training kernel.test(x_train,y_train,32) #test the network performance on the train set with batch size 32 @@ -141,9 +141,9 @@ x_train=x_train.reshape([60000,784]) #reshape data to fit the network input nn=n.nn() #create neural network object nn.build() #build the network structure kernel=k.kernel(nn) #create kernel object with the network -kernel.process=7 #set the number of processes to train +kernel.process=3 #set the number of processes to train kernel.data_segment_flag=True #set the flag to segment data for each process -kernel.epoch=6 #set the number of epochs to train +kernel.epoch=5 #set the number of epochs to train kernel.batch=32 #set the batch size kernel.PO=2 #use PO2 algorithm for parallel optimization kernel.data(x_train,y_train) #input train data to the kernel @@ -151,7 +151,7 @@ manager=Manager() #create manager object to share data among kernel.init(manager) #initialize shared data with the manager lock=[Lock(),Lock()] #create two locks for synchronization g_lock=Lock() #create a global lock for gradient computing -for p in range(7): #loop over the processes +for p in range(3): #loop over the processes Process(target=kernel.train,args=(p,lock,g_lock)).start() #start each process with the train function and pass the process id and locks as arguments kernel.update_nn_param() #update the network parameters after training kernel.test(x_train,y_train,32) #test the network performance on the train set with batch size 32 @@ -168,9 +168,9 @@ x_train=x_train.reshape([60000,784]) #reshape data to fit the network input nn=n.nn() #create neural network object nn.build() #build the network structure kernel=k.kernel(nn) #create kernel object with the network -kernel.process=7 #set the number of processes to train +kernel.process=3 #set the number of processes to train kernel.data_segment_flag=True #set the flag to segment data for each process -kernel.epoch=6 #set the number of epochs to train +kernel.epoch=5 #set the number of epochs to train kernel.batch=32 #set the batch size kernel.PO=2 #use PO2 algorithm for parallel optimization kernel.data(x_train,y_train) #input train data to the kernel @@ -178,7 +178,7 @@ manager=Manager() #create manager object to share data among kernel.init(manager) #initialize shared data with the manager lock=[Lock(),Lock()] #create two locks for synchronization g_lock=[Lock(),Lock()] #create a list of global locks for gradient computing -for p in range(7): #loop over the processes +for p in range(3): #loop over the processes Process(target=kernel.train,args=(p,lock,g_lock)).start() #start each process with the train function and pass the process id, the locks and the global locks as arguments kernel.update_nn_param() #update the network parameters after training kernel.test(x_train,y_train,32) #test the network performance on the train set with batch size 32 @@ -196,16 +196,16 @@ x_train,x_test =x_train/255.0,x_test/255.0 #normalize data nn=n.nn() #create neural network object nn.build() #build the network structure kernel=k.kernel(nn) #create kernel object with the network -kernel.process=7 #set the number of processes to train +kernel.process=3 #set the number of processes to train kernel.data_segment_flag=True #set the flag to segment data for each process -kernel.epoch=6 #set the number of epochs to train +kernel.epoch=5 #set the number of epochs to train kernel.batch=32 #set the batch size kernel.PO=3 #use PO3 algorithm for parallel optimization kernel.data(x_train,y_train) #input train data to the kernel manager=Manager() #create manager object to share data among processes kernel.init(manager) #initialize shared data with the manager lock=Lock() #create a lock for synchronization -for p in range(7): #loop over the processes +for p in range(3): #loop over the processes Process(target=kernel.train,args=(p,lock)).start() #start each process with the train function and pass the process id and the lock as arguments kernel.update_nn_param() #update the network parameters after training kernel.test(x_train,y_train,32) #test the network performance on the train set with batch size 32 @@ -223,15 +223,15 @@ x_train,x_test =x_train/255.0,x_test/255.0 #normalize data nn=n.nn() #create neural network object nn.build() #build the network structure kernel=k.kernel(nn) #create kernel object with the network -kernel.process=7 #set the number of processes to train +kernel.process=3 #set the number of processes to train kernel.data_segment_flag=True #set the flag to segment data for each process -kernel.epoch=6 #set the number of epochs to train +kernel.epoch=5 #set the number of epochs to train kernel.batch=32 #set the batch size kernel.PO=3 #use PO3 algorithm for parallel optimization kernel.data(x_train,y_train) #input train data to the kernel manager=Manager() #create manager object to share data among processes kernel.init(manager) #initialize shared data with the manager -for p in range(7): #loop over the processes +for p in range(3): #loop over the processes Process(target=kernel.train,args=(p,)).start() #start each process with the train function and pass the process id as argument kernel.update_nn_param() #update the network parameters after training kernel.test(x_train,y_train,32) #test the network performance on the train set with batch size 32 @@ -246,7 +246,7 @@ mnist=tf.keras.datasets.mnist #load mnist dataset (x_train,y_train),(x_test,y_test)=mnist.load_data() #split data into train and test sets x_train,x_test =x_train/255.0,x_test/255.0 #normalize data kernel=k.kernel() #create kernel object without a network -kernel.process=7 #set the number of processes to train +kernel.process=3 #set the number of processes to train kernel.data_segment_flag=True #set the flag to segment data for each process kernel.epoch=1 #set the number of epochs to train kernel.batch=32 #set the batch size @@ -255,7 +255,7 @@ kernel.data(x_train,y_train) #input train data to the kernel manager=Manager() #create manager object to share data among processes kernel.init(manager) #initialize shared data with the manager kernel.restore('save.dat') #restore the neural network from a file -for p in range(7): #loop over the processes +for p in range(3): #loop over the processes Process(target=kernel.train,args=(p,)).start() #start each process with the train function and pass the process id as argument ``` @@ -271,16 +271,16 @@ x_train,x_test =x_train/255.0,x_test/255.0 #normalize data nn=n.nn() #create neural network object nn.build() #build the network structure kernel=k.kernel(nn) #create kernel object with the network -kernel.process=7 #set the number of processes to train +kernel.process=3 #set the number of processes to train kernel.process_t=3 #set the number of processes to test kernel.data_segment_flag=True #set the flag to segment data for each process -kernel.epoch=6 #set the number of epochs to train +kernel.epoch=5 #set the number of epochs to train kernel.batch=32 #set the batch size kernel.PO=3 #use PO3 algorithm for parallel optimization kernel.data(x_train,y_train,x_test,y_test) #input train and test data to the kernel manager=Manager() #create manager object to share data among processes kernel.init(manager) #initialize shared data with the manager -for p in range(7): #loop over the processes +for p in range(3): #loop over the processes Process(target=kernel.train,args=(p,None,None,32)).start() #start each process with the train function and pass the process id, the locks, the test flag and the test batch size as arguments ``` @@ -299,15 +299,15 @@ nn.build() #build the network structure kernel=k.kernel(nn) #create kernel object with the network kernel.stop=True #set the flag to stop training when a condition is met kernel.end_loss=0.7 #set the condition to stop training when the loss is less than 0.7 -kernel.process=7 #set the number of processes to train +kernel.process=3 #set the number of processes to train kernel.data_segment_flag=True #set the flag to segment data for each process -kernel.epoch=6 #set the number of epochs to train +kernel.epoch=5 #set the number of epochs to train kernel.batch=32 #set the batch size kernel.PO=3 #use PO3 algorithm for parallel optimization kernel.data(x_train,y_train) #input train data to the kernel manager=Manager() #create manager object to share data among processes kernel.init(manager) #initialize shared data with the manager -for p in range(7): #loop over the processes +for p in range(3): #loop over the processes Process(target=kernel.train,args=(p,)).start() #start each process with the train function and pass the process id and locks as arguments ``` From e8e2355e23846ead253e3f25f0b813291682d3a5 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Fri, 28 Jul 2023 23:16:10 +0800 Subject: [PATCH 137/337] Update nn_attenuate.py --- .../DL/neural network/tensorflow/parallel/nn_attenuate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn_attenuate.py b/Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn_attenuate.py index 3f09c8b8..d57ee416 100644 --- a/Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn_attenuate.py +++ b/Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn_attenuate.py @@ -12,7 +12,7 @@ def __init__(self): self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) self.optimizer=Momentum(0.07,0.7) # Initialize a variable to keep track of the number of optimization steps - self.opt_counter=tf.Variable(tf.zeros(7,dtype=tf.float32)) + self.opt_counter=tf.Variable(tf.zeros(3,dtype=tf.float32)) def build(self): From 967485f7283be08911fb145093dac490ed5288d1 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Sat, 29 Jul 2023 19:36:21 +0800 Subject: [PATCH 138/337] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 12ccd2f3..687e72b7 100644 --- a/README.md +++ b/README.md @@ -412,7 +412,7 @@ test=t.parallel_test(nn,x_test,y_test,6,32) #create parallel test object with th test.segment_data() #segment data for each process for p in range(6): #loop over the processes Process(target=test.test).start() #start each process with the test function -loss=test.loss_acc() #calculate the loss and accuracy of the test +loss,acc=test.loss_acc() #calculate the loss and accuracy of the test ``` From f38fb11170698b39282b2e1869f1800e6e4a9530 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Sat, 29 Jul 2023 23:07:09 +0800 Subject: [PATCH 139/337] Update README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 687e72b7..8cf41b26 100644 --- a/README.md +++ b/README.md @@ -307,8 +307,9 @@ kernel.PO=3 #use PO3 algorithm for parallel optimizatio kernel.data(x_train,y_train) #input train data to the kernel manager=Manager() #create manager object to share data among processes kernel.init(manager) #initialize shared data with the manager +lock=Lock() #create a lock for synchronization for p in range(3): #loop over the processes - Process(target=kernel.train,args=(p,)).start() #start each process with the train function and pass the process id and locks as arguments + Process(target=kernel.train,args=(p,lock)).start() #start each process with the train function and pass the process id and locks as arguments ``` ## RL： From e512439bd04f975f1743bbff04922bf9b635091e Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Mon, 31 Jul 2023 22:15:39 +0800 Subject: [PATCH 140/337] Update nn_device.py --- .../DL/neural network/tensorflow/parallel/nn_device.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn_device.py b/Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn_device.py index 16ef49eb..b9408b17 100644 --- a/Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn_device.py +++ b/Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn_device.py @@ -9,7 +9,6 @@ class nn: # A neural network class example, allocate device for mu def __init__(self): # initialize the network self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) # loss object, kernel uses it to calculate loss. Here we use sparse categorical crossentropy with logits as output self.optimizer=Momentum(0.07,0.7) # optimizer, kernel uses it to optimize. Here we use a custom momentum optimizer with learning rate 0.07 and momentum 0.7 - self.info='example' # some information about the network def build(self): # build function, kernel uses it to create the network layers @@ -37,4 +36,4 @@ def loss(self,output,labels,p): # loss function, kernel uses it to calculate los def opt(self,gradient): # optimization function, kernel uses it to optimize parameter # Perform optimization on the parameters using the gradient param=self.optimizer.opt(gradient,self.param) # apply the Note's momentum optimizer to update the parameters using the gradient - return param \ No newline at end of file + return param From 3517b08669dd40427d0477b66aa7d5a0120a7dca Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Tue, 1 Aug 2023 12:44:36 +0800 Subject: [PATCH 141/337] Update README.md --- README.md | 9 --------- 1 file changed, 9 deletions(-) diff --git a/README.md b/README.md index 8cf41b26..a6c06732 100644 --- a/README.md +++ b/README.md @@ -461,12 +461,3 @@ import DQN as d #import deep Q-network module dqn=d.DQN(4,128,2) #create neural network object with 4 inputs, 128 hidden units and 2 outputs check(dqn,tf,2) #check the network with tensorflow platform and 2 actions ``` - - -# Note Compiler: -documentation:https://github.com/NoteDancing/Note-documentation/tree/Note-7.0/Note%207.0%20documentation/compiler -```python -import Note.nc as nc -c=nc.compiler('nn.n') -c.Compile() -``` From 3bcc9349b473f94c0f3f59548cebbfc1ee8caba1 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Tue, 1 Aug 2023 18:54:34 +0800 Subject: [PATCH 142/337] Update nn_device.py --- .../DL/neural network/tensorflow/parallel/nn_device.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn_device.py b/Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn_device.py index b9408b17..c5dd9156 100644 --- a/Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn_device.py +++ b/Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn_device.py @@ -33,7 +33,8 @@ def loss(self,output,labels,p): # loss function, kernel uses it to calculate los return self.loss_object(labels,output) # return the mean softmax cross entropy loss between labels and output - def opt(self,gradient): # optimization function, kernel uses it to optimize parameter + def opt(self,gradient,p): # optimization function, kernel uses it to optimize parameter # Perform optimization on the parameters using the gradient - param=self.optimizer.opt(gradient,self.param) # apply the Note's momentum optimizer to update the parameters using the gradient + with tf.device(assign_device(p,'GPU')): # assign the device according to the process index p + param=self.optimizer.opt(gradient,self.param) # apply the Note's momentum optimizer to update the parameters using the gradient return param From 07d80b93ceb865208c879786f7cb9066cfbb892f Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Tue, 1 Aug 2023 18:56:28 +0800 Subject: [PATCH 143/337] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a6c06732..ed6c6779 100644 --- a/README.md +++ b/README.md @@ -216,7 +216,7 @@ kernel.test(x_train,y_train,32) #test the network performance on the train import Note.DL.parallel.kernel as k #import kernel module import tensorflow as tf #import tensorflow library import nn as n #import neural network module -from multiprocessing import Process,Lock,Manager #import multiprocessing tools +from multiprocessing import Process,Manager #import multiprocessing tools mnist=tf.keras.datasets.mnist #load mnist dataset (x_train,y_train),(x_test,y_test)=mnist.load_data() #split data into train and test sets x_train,x_test =x_train/255.0,x_test/255.0 #normalize data @@ -241,7 +241,7 @@ kernel.save() #save the neural network to a file import Note.DL.parallel.kernel as k #import kernel module import tensorflow as tf #import tensorflow library import nn as n #import neural network module -from multiprocessing import Process,Lock,Manager #import multiprocessing tools +from multiprocessing import Process,Manager #import multiprocessing tools mnist=tf.keras.datasets.mnist #load mnist dataset (x_train,y_train),(x_test,y_test)=mnist.load_data() #split data into train and test sets x_train,x_test =x_train/255.0,x_test/255.0 #normalize data From 802eebb6b405555fa29d2ef673f09df2bd8e16f5 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Wed, 2 Aug 2023 14:46:39 +0800 Subject: [PATCH 144/337] Update kernel_pytorch.py --- Note 7.0 documentation/RL/kernel/parallel/kernel_pytorch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Note 7.0 documentation/RL/kernel/parallel/kernel_pytorch.py b/Note 7.0 documentation/RL/kernel/parallel/kernel_pytorch.py index 7c2f512f..25cc2d67 100644 --- a/Note 7.0 documentation/RL/kernel/parallel/kernel_pytorch.py +++ b/Note 7.0 documentation/RL/kernel/parallel/kernel_pytorch.py @@ -1,6 +1,6 @@ import torch from multiprocessing import Value,Array -from Note.nn.process.assign_device_pytorch import assign_device +from Note.nn.parallel.assign_device_pytorch import assign_device import numpy as np import statistics From 98790d0dd32c75706d392ef14f48487c22d1aabe Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Wed, 2 Aug 2023 21:19:47 +0800 Subject: [PATCH 145/337] Update README.md --- README.md | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index ed6c6779..79bd24fe 100644 --- a/README.md +++ b/README.md @@ -326,6 +326,7 @@ import DQN as d #import deep Q-network module from multiprocessing import Process,Lock,Manager #import multiprocessing tools dqn=d.DQN(4,128,2) #create neural network object with 4 inputs, 128 hidden units and 2 outputs kernel=k.kernel(dqn,5) #create kernel object with the network and 5 processes to train +kernel.episode=300 #set the number of episodes to 300 manager=Manager() #create manager object to share data among processes kernel.init(manager) #initialize shared data with the manager kernel.action_count=2 #set the number of actions to 2 @@ -334,7 +335,7 @@ kernel.PO=1 #use PO1 algorithm for parallel optimization pool_lock=[Lock(),Lock(),Lock(),Lock(),Lock()] #create a list of locks for each process's replay pool lock=[Lock(),Lock(),Lock()] #create a list of locks for synchronization for p in range(5): #loop over the processes - Process(target=kernel.train,args=(p,100,lock,pool_lock)).start() #start each process with the train function and pass the process id, the number of episodes, the locks and the pool locks as arguments + Process(target=kernel.train,args=(p,lock,pool_lock)).start() #start each process with the train function and pass the process id, the number of episodes, the locks and the pool locks as arguments ``` ### PO2: @@ -344,6 +345,7 @@ import DQN as d #import deep Q-network module from multiprocessing import Process,Lock,Manager #import multiprocessing tools dqn=d.DQN(4,128,2) #create neural network object with 4 inputs, 128 hidden units and 2 outputs kernel=k.kernel(dqn,5) #create kernel object with the network and 5 processes to train +kernel.episode=300 #set the number of episodes to 300 manager=Manager() #create manager object to share data among processes kernel.init(manager) #initialize shared data with the manager kernel.action_count=2 #set the number of actions to 2 @@ -353,7 +355,7 @@ pool_lock=[Lock(),Lock(),Lock(),Lock(),Lock()] #create a list of locks for each lock=[Lock(),Lock(),Lock()] #create a list of locks for synchronization g_lock=Lock() #create a global lock for gradient computing for p in range(5): #loop over the processes - Process(target=kernel.train,args=(p,100,lock,pool_lock,g_lock)).start() #start each process with the train function and pass the process id, the number of episodes, the locks, the pool locks and the global lock as arguments + Process(target=kernel.train,args=(p,lock,pool_lock,g_lock)).start() #start each process with the train function and pass the process id, the number of episodes, the locks, the pool locks and the global lock as arguments ``` ### PO3: @@ -363,6 +365,7 @@ import DQN as d #import deep Q-network module from multiprocessing import Process,Lock,Manager #import multiprocessing tools dqn=d.DQN(4,128,2) #create neural network object with 4 inputs, 128 hidden units and 2 outputs kernel=k.kernel(dqn,5) #create kernel object with the network and 5 processes to train +kernel.episode=300 #set the number of episodes to 300 manager=Manager() #create manager object to share data among processes kernel.init(manager) #initialize shared data with the manager kernel.action_count=2 #set the number of actions to 2 @@ -371,7 +374,7 @@ kernel.PO=3 #use PO3 algorithm for parallel optimization pool_lock=[Lock(),Lock(),Lock(),Lock(),Lock()] #create a list of locks for each process's replay pool lock=[Lock(),Lock(),Lock()] #create three locks for synchronization for p in range(5): #loop over the processes - Process(target=kernel.train,args=(p,100,lock,pool_lock)).start() #start each process with the train function and pass the process id, the number of episodes, the locks and the pool locks as arguments + Process(target=kernel.train,args=(p,lock,pool_lock)).start() #start each process with the train function and pass the process id, the number of episodes, the locks and the pool locks as arguments ``` ### Stop training and saving when condition is met: @@ -381,6 +384,7 @@ import DQN as d #import deep Q-network module from multiprocessing import Process,Lock,Manager #import multiprocessing tools dqn=d.DQN(4,128,2) #create neural network object with 4 inputs, 128 hidden units and 2 outputs kernel=k.kernel(dqn,5) #create kernel object with the network and 5 processes to train +kernel.episode=300 #set the number of episodes to 300 manager=Manager() #create manager object to share data among processes kernel.init(manager) #initialize shared data with the manager kernel.action_count=2 #set the number of actions to 2 @@ -390,7 +394,7 @@ kernel.PO=3 #use PO3 algorithm for parallel optimization pool_lock=[Lock(),Lock(),Lock(),Lock(),Lock()] #create a list of locks for each process's replay pool lock=[Lock(),Lock(),Lock()] #create three locks for synchronization for p in range(5): #loop over the processes - Process(target=kernel.train,args=(p,100,lock,pool_lock)).start() #start each process with the train function and pass the process id, the number of episodes, the locks and the pool locks as arguments + Process(target=kernel.train,args=(p,lock,pool_lock)).start() #start each process with the train function and pass the process id, the number of episodes, the locks and the pool locks as arguments ``` From 7a7746ed7f73ddc3b363156a34c401e2a407f782 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Wed, 2 Aug 2023 22:16:05 +0800 Subject: [PATCH 146/337] Update kernel.py --- .../RL/kernel/parallel/kernel.py | 1068 +++++++++-------- 1 file changed, 545 insertions(+), 523 deletions(-) diff --git a/Note 7.0 documentation/RL/kernel/parallel/kernel.py b/Note 7.0 documentation/RL/kernel/parallel/kernel.py index 4d07e1c3..ccf22769 100644 --- a/Note 7.0 documentation/RL/kernel/parallel/kernel.py +++ b/Note 7.0 documentation/RL/kernel/parallel/kernel.py @@ -1,6 +1,3 @@ -# This is a code snippet that defines a kernel class for multi-process reinforcement learning -# using TensorFlow and multiprocessing - import tensorflow as tf from tensorflow.python.ops import state_ops from tensorflow.python.util import nest @@ -11,630 +8,655 @@ class kernel: def __init__(self,nn=None,process=None): - self.nn=nn # the neural network model - if process!=None: + self.nn=nn # the neural network model to be trained + if process!=None: # the number of processes to be used for parallel training self.reward=np.zeros(process,dtype=np.float32) # the reward array for each process self.sc=np.zeros(process,dtype=np.int32) # the step counter array for each process - self.epsilon=None # the exploration rate array for each process - self.episode_step=None # the maximum number of steps per episode - self.pool_size=None # the maximum size of the pool - self.batch=None # the batch size for training - self.update_step=None # the update frequency for the target network - self.trial_count=None # the number of trials to compute the average reward - self.process=process # the number of processes - self.PO=None # the parallel optimization mode (1, 2, or 3) - self.priority_flag=False # the priority flag for optimization order - self.max_opt=None # the maximum number of optimization steps per process per episode - self.stop=False # the stop flag for training - self.s=None # the state array for online training mode - self.filename='save.dat' # the file name to save parameters to file + self.epsilon=None # the epsilon value for epsilon-greedy policy + self.episode_step=None # the maximum number of steps for each episode + self.pool_size=None # the size of the memory pool for storing experience data + self.episode=None # the number of episodes for training + self.batch=None # the size of the batch for training + self.update_step=None # the frequency of updating the neural network parameters + self.trial_count=None # the number of trials to calculate the average reward + self.criterion=None # the criterion for stopping the training based on the average reward + self.process=process # the number of processes to be used for parallel training + self.PO=None # the parallel optimization mode (1, 2 or 3) + self.priority_flag=False # a flag indicating whether to use priority optimization or not + self.max_opt=None # the maximum number of optimization steps for priority optimization + self.stop=False # a flag indicating whether to stop the training or not + self.opt_counter=None # a counter array for tracking the optimization steps for each process + self.s=None # a state variable for online training + self.filename='save.dat' # a file name for saving the neural network parameters def init(self,manager): - self.state_pool=manager.dict({}) # create a shared memory space for state pool using manager object - self.action_pool=manager.dict({}) # create a shared memory space for action pool using manager object - self.next_state_pool=manager.dict({}) # create a shared memory space for next state pool using manager object - self.reward_pool=manager.dict({}) # create a shared memory space for reward pool using manager object - self.done_pool=manager.dict({}) # create a shared memory space for done pool using manager object - self.reward=Array('f',self.reward) # create a shared memory space for reward array using Array object + self.state_pool=manager.dict({}) # a shared dictionary for storing state data for each process + self.action_pool=manager.dict({}) # a shared dictionary for storing action data for each process + self.next_state_pool=manager.dict({}) # a shared dictionary for storing next state data for each process + self.reward_pool=manager.dict({}) # a shared dictionary for storing reward data for each process + self.done_pool=manager.dict({}) # a shared dictionary for storing done flag data for each process + self.reward=Array('f',self.reward) # a shared array for storing reward data for each process if type(self.nn.param[0])!=list: - self.loss=np.zeros(self.process,dtype=self.nn.param[0].dtype.name) # create a loss array with the same data type as neural network parameters + self.loss=np.zeros(self.process,dtype=self.nn.param[0].dtype.name) # a loss array for each process else: - self.loss=np.zeros(self.process,dtype=self.nn.param[0][0].dtype.name) # create a loss array with the same data type as neural network parameters - self.loss=Array('f',self.loss) - self.sc=Array('i',self.sc) # create a shared memory space for step counter array using Array object - self.process_counter=Value('i',0) # create a shared memory space for process counter using Value object - self.probability_list=manager.list([]) # create a shared memory space for probability list using manager object - self.running_flag_list=manager.list([]) # create a shared memory space for running flag list using manager object - self.finish_list=manager.list([]) # create a shared memory space for finish flag list using manager object - self.running_flag=manager.list([0]) # create a shared memory space for running flag array using manager object - self.reward_list=manager.list([]) # create a shared memory space for reward list using manager object - self.loss_list=manager.list([]) # create a shared memory space for loss list using manager object - self.total_episode=Value('i',0) # create a shared memory space for total episode counter using Value object - self.priority_p=Value('i',0) # create a shared memory space for priority index using Value object - if self.priority_flag==True: - self.opt_counter=Array('i',np.zeros(self.process,dtype=np.int32)) # create a shared memory space for optimization counter array using Array object + self.loss=np.zeros(self.process,dtype=self.nn.param[0][0].dtype.name) # a loss array for each process + self.loss=Array('f',self.loss) # a shared array for storing loss data for each process + self.sc=Array('i',self.sc) # a shared array for storing step counter data for each process + self.process_counter=Value('i',0) # a shared value for counting the number of running processes + self.probability_list=manager.list([]) # a shared list for storing probability data for each process + self.running_flag_list=manager.list([]) # a shared list for storing running flag data for each process + self.finish_list=manager.list([]) # a shared list for storing the finished process indices + self.running_flag=manager.list([0]) # a shared list for storing the running flag for each process + self.reward_list=manager.list([]) # a shared list for storing the total reward for each episode + self.loss_list=manager.list([]) # a shared list for storing the average loss for each episode + self.episode_counter=Value('i',0) # a shared value for counting the number of completed episodes + self.total_episode=Value('i',0) # a shared value for counting the total number of episodes + self.priority_p=Value('i',0) # a shared value for storing the priority process index + if self.priority_flag==True: # if priority optimization is enabled + self.opt_counter=Array('i',np.zeros(self.process,dtype=np.int32)) # a shared array for storing the optimization counter for each process try: - self.nn.opt_counter=manager.list([self.nn.opt_counter]) # create a shared memory space for neural network optimization counter using manager object + self.nn.opt_counter=manager.list([self.nn.opt_counter]) # a shared list for storing the optimization counter for the neural network model except Exception: - self.opt_counter_=manager.list() # create an empty list to store the exception + self.opt_counter_=manager.list() # a dummy list for handling exception try: - self.nn.ec=manager.list([self.nn.ec]) # create a shared memory space for neural network episode counter using manager object + self.nn.ec=manager.list([self.nn.ec]) # a shared list for storing the episode counter for the neural network model except Exception: - self.ec_=manager.list() # create an empty list to store the exception + self.ec_=manager.list() # a dummy list for handling exception try: - self.nn.bc=manager.list([self.nn.bc]) # create a shared memory space for neural network batch counter using manager object + self.nn.bc=manager.list([self.nn.bc]) # a shared list for storing the batch counter for the neural network model except Exception: - self.bc_=manager.list() # create an empty list to store the exception - self.episode_=Value('i',self.total_episode) # create a shared memory space for episode counter using Value object - self.stop_flag=Value('b',False) # create a shared memory space for stop flag using Value object - self.save_flag=Value('b',False) # create a shared memory space for save flag using Value object - self.file_list=manager.list([]) # create an empty list to store the file names - self.param=manager.dict() # create an empty dictionary to store the parameters + self.bc_=manager.list() # a dummy list for handling exception + self.episode_=Value('i',self.total_episode.value) # a copy of the total episode value + self.stop_flag=Value('b',False) # a shared flag for indicating whether to stop the training or not + self.save_flag=Value('b',False) # a shared flag for indicating whether to save the neural network parameters or not + self.file_list=manager.list([]) # a shared list for storing the file names for saving the neural network parameters + self.param=manager.dict() # a shared dictionary for storing the neural network parameters + self.param[7]=self.nn.param # assign the neural network parameters to the dictionary with key 7 return def init_online(self,manager): - self.nn.train_loss_list=manager.list([]) # create an empty list to store the training losses - self.nn.counter=manager.list([]) # create an empty list to store the training counters - self.nn.exception_list=manager.list([]) # create an empty list to store the exceptions - self.param=manager.dict() # create an empty dictionary to store the parameters - self.param[7]=self.nn.param # assign the neural network parameters to the dictionary + self.nn.train_loss_list=manager.list([]) # a shared list for storing the training loss data for online training + self.nn.counter=manager.list([]) # a shared list for storing the counter data for online training + self.nn.exception_list=manager.list([]) # a shared list for storing the exception data for online training + self.param=manager.dict() # a shared dictionary for storing the neural network parameters + self.param[7]=self.nn.param # assign the neural network parameters to the dictionary with key 7 return def action_vec(self): - self.action_one=np.ones(self.action_count,dtype=np.int8) # create an action vector with ones + self.action_one=np.ones(self.action_count,dtype=np.int8) # an array of ones with length equal to the action count return def set_up(self,epsilon=None,episode_step=None,pool_size=None,batch=None,update_step=None,trial_count=None,criterion=None): - if epsilon!=None: - self.epsilon=np.ones(self.process)*epsilon # assign the exploration rate array with epsilon - if episode_step!=None: - self.episode_step=episode_step # assign the maximum number of steps per episode - if pool_size!=None: - self.pool_size=pool_size # assign the maximum size of the pool - if batch!=None: - self.batch=batch # assign the batch size for training - if update_step!=None: - self.update_step=update_step # assign the update frequency for the target network - if trial_count!=None: - self.trial_count=trial_count # assign the number of trials to compute the average reward - if criterion!=None: - self.criterion=criterion # assign the criterion to judge if the agent solves the environment - if epsilon!=None: - self.action_vec() # create an action vector with ones + if epsilon!=None: # if epsilon value is given + self.epsilon=np.ones(self.process)*epsilon # assign epsilon value to an array with length equal to the process count + self.action_vec() # call the action_vec method to create an action array + if episode_step!=None: # if episode step value is given + self.episode_step=episode_step # assign episode step value to an attribute + if pool_size!=None: # if pool size value is given + self.pool_size=pool_size # assign pool size value to an attribute + if batch!=None: # if batch size value is given + self.batch=batch # assign batch size value to an attribute + if update_step!=None: # if update step value is given + self.update_step=update_step # assign update step value to an attribute + if trial_count!=None: # if trial count value is given + self.trial_count=trial_count # assign trial count value to an attribute + if criterion!=None: # if criterion value is given + self.criterion=criterion # assign criterion value to an attribute return def epsilon_greedy_policy(self,s,epsilon): - action_prob=self.action_one*epsilon/len(self.action_one) # create a uniform action probability vector based on epsilon - best_a=np.argmax(self.nn.nn.fp(s)) # get the best action based on the neural network output + action_prob=self.action_one*epsilon/len(self.action_one) # calculate the action probability based on epsilon and action array + best_a=np.argmax(self.nn.nn.fp(s)) # find the best action index based on the neural network output action_prob[best_a]+=1-epsilon # increase the probability of the best action by 1-epsilon - return action_prob # return the action probability vector + return action_prob # return the action probability array def pool(self,s,a,next_s,r,done,pool_lock,index): - pool_lock[index].acquire() # acquire the lock for the pool of the current process + pool_lock[index].acquire() # acquire the lock for the given index try: - if type(self.state_pool[index])!=np.ndarray and self.state_pool[index]==None: # check if the pool is empty - self.state_pool[index]=s # assign the state to the pool - if type(a)==int: # check if the action is an integer - a=np.array(a) # convert the action to an array - self.action_pool[index]=np.expand_dims(a,axis=0) # expand the dimension of the action and assign it to the pool - else: # otherwise, assume the action is an array - self.action_pool[index]=a # assign the action to the pool - self.next_state_pool[index]=np.expand_dims(next_s,axis=0) # expand the dimension of the next state and assign it to the pool - self.reward_pool[index]=np.expand_dims(r,axis=0) # expand the dimension of the reward and assign it to the pool - self.done_pool[index]=np.expand_dims(done,axis=0) # expand the dimension of the done flag and assign it to the pool - else: # otherwise, assume the pool is not empty + if type(self.state_pool[index])!=np.ndarray and self.state_pool[index]==None: # if the state pool for the given index is empty + self.state_pool[index]=s # assign the state data to the state pool + if type(a)==int: # if the action data is an integer + a=np.array(a) # convert it to a numpy array + self.action_pool[index]=np.expand_dims(a,axis=0) # add a dimension and assign it to the action pool + else: # if the action data is not an integer + self.action_pool[index]=a # assign it to the action pool directly + self.next_state_pool[index]=np.expand_dims(next_s,axis=0) # add a dimension and assign the next state data to the next state pool + self.reward_pool[index]=np.expand_dims(r,axis=0) # add a dimension and assign the reward data to the reward pool + self.done_pool[index]=np.expand_dims(done,axis=0) # add a dimension and assign the done flag data to the done pool + else: # if the state pool for the given index is not empty try: - self.state_pool[index]=np.concatenate((self.state_pool[index],s),0) # concatenate the state with the existing pool - if type(a)==int: # check if the action is an integer - a=np.array(a) # convert the action to an array - self.action_pool[index]=np.concatenate((self.action_pool[index],np.expand_dims(a,axis=0)),0) # expand and concatenate the action with the existing pool - else: # otherwise, assume the action is an array - self.action_pool[index]=np.concatenate((self.action_pool[index],a),0) # concatenate the action with the existing pool - self.next_state_pool[index]=np.concatenate((self.next_state_pool[index],np.expand_dims(next_s,axis=0)),0) # expand and concatenate the next state with the existing pool - self.reward_pool[index]=np.concatenate((self.reward_pool[index],np.expand_dims(r,axis=0)),0) # expand and concatenate the reward with the existing pool - self.done_pool[index]=np.concatenate((self.done_pool[index],np.expand_dims(done,axis=0)),0) # expand and concatenate the done flag with the existing pool + self.state_pool[index]=np.concatenate((self.state_pool[index],s),0) # concatenate the state data to the state pool + if type(a)==int: # if the action data is an integer + a=np.array(a) # convert it to a numpy array + self.action_pool[index]=np.concatenate((self.action_pool[index],np.expand_dims(a,axis=0)),0) # add a dimension and concatenate it to the action pool + else: # if the action data is not an integer + self.action_pool[index]=np.concatenate((self.action_pool[index],a),0) # concatenate it to the action pool directly + self.next_state_pool[index]=np.concatenate((self.next_state_pool[index],np.expand_dims(next_s,axis=0)),0) # add a dimension and concatenate the next state data to the next state pool + self.reward_pool[index]=np.concatenate((self.reward_pool[index],np.expand_dims(r,axis=0)),0) # add a dimension and concatenate the reward data to the reward pool + self.done_pool[index]=np.concatenate((self.done_pool[index],np.expand_dims(done,axis=0)),0) # add a dimension and concatenate the done flag data to the done pool except Exception: - pass # ignore any exception - if type(self.state_pool[index])==np.ndarray and len(self.state_pool[index])>self.pool_size: # check if the pool size exceeds the maximum size - self.state_pool[index]=self.state_pool[index][1:] # delete the oldest state from the pool - self.action_pool[index]=self.action_pool[index][1:] # delete the oldest action from the pool - self.next_state_pool[index]=self.next_state_pool[index][1:] # delete the oldest next state from the pool - self.reward_pool[index]=self.reward_pool[index][1:] # delete the oldest reward from the pool - self.done_pool[index]=self.done_pool[index][1:] # delete the oldest done flag from the pool + pass # ignore any exception that may occur during concatenation + if type(self.state_pool[index])==np.ndarray and len(self.state_pool[index])>self.pool_size: # if the state pool for the given index exceeds the pool size limit + self.state_pool[index]=self.state_pool[index][1:] # remove the oldest state data from the state pool + self.action_pool[index]=self.action_pool[index][1:] # remove the oldest action data from the action pool + self.next_state_pool[index]=self.next_state_pool[index][1:] # remove the oldest next state data from the next state pool + self.reward_pool[index]=self.reward_pool[index][1:] # remove the oldest reward data from the reward pool + self.done_pool[index]=self.done_pool[index][1:] # remove the oldest done flag data from the done pool except Exception: - pool_lock[index].release() # release the lock for the pool of the current process - return - pool_lock[index].release() # release the lock for the pool of the current process + pool_lock[index].release() # release the lock for the given index in case of any exception + return + pool_lock[index].release() # release the lock for the given index after storing all data in pools return def get_index(self,p,lock): - while len(self.running_flag_list)self.process_counter.value: # check if running flag list needs to be updated - self.running_flag_list[p]=self.running_flag[1:].copy() # update running flag list with a copy of running flag array - while len(self.probability_list)self.process_counter.value: # if the running flag list for the given p is not consistent with the process counter value + self.running_flag_list[p]=self.running_flag[1:].copy() # update the running flag list for the given p with a copy of the running flag list without the first element + while len(self.probability_list)=self.trial_count: # check if enough rewards are available - avg_reward=statistics.mean(self.reward_list[-self.trial_count:]) # compute the average reward of the last trial count episodes - if self.criterion!=None and avg_reward>=self.criterion: # check if criterion is set and average reward is greater than or equal to criterion - return True # return True to indicate that the agent solves the environment - return False # return False to indicate that the agent does not solve the environment + if self.trial_count!=None: # if trial count value is given + if len(self.reward_list)>=self.trial_count: # if reward list has enough elements + avg_reward=statistics.mean(self.reward_list[-self.trial_count:]) # calculate the average reward of the last trial count elements + if self.criterion!=None and avg_reward>=self.criterion: # if criterion value is given and average reward is greater than or equal to criterion value + return True # return True, indicating that training should end + return False # otherwise, return False, indicating that training should continue - @tf.function(jit_compile=True) + @tf.function(jit_compile=True) # use tensorflow function decorator with jit_compile option to improve computation efficiency and automatic differentiation def opt(self,state_batch,action_batch,next_state_batch,reward_batch,done_batch,p,lock,g_lock=None): - with tf.GradientTape(persistent=True) as tape: # create a gradient tape object to record gradients + with tf.GradientTape(persistent=True) as tape: # use tensorflow gradient tape with persistent option to record operations for automatic differentiation try: try: - loss=self.nn.loss(state_batch,action_batch,next_state_batch,reward_batch,done_batch) # compute the loss using neural network model + loss=self.nn.loss(state_batch,action_batch,next_state_batch,reward_batch,done_batch) # calculate loss by calling loss attribute of neural network model with batch data as arguments except Exception: - loss=self.nn.loss(state_batch,action_batch,next_state_batch,reward_batch,done_batch,p) # compute the loss using neural network model and process index + loss=self.nn.loss(state_batch,action_batch,next_state_batch,reward_batch,done_batch,p) # if exception occurs, try again with process index as an additional argument except Exception as e: - raise e # raise any exception - if self.PO==1: # check if parallel optimization mode is 1 - if self.priority_flag==True and self.priority_p.value!=-1: # check if priority flag is True and priority index is not -1 + raise e # if exception still occurs, raise it + if self.PO==1: # if parallel optimization mode is 1 + if self.priority_flag==True and self.priority_p.value!=-1: # if priority optimization is enabled and priority process index is not -1 while True: - if p==self.priority_p.value: # check if current process index is equal to priority index - break # break the loop - else: - continue # try again - lock[0].acquire() # acquire the lock for optimization operation - if self.stop_func_(lock[0]): # check if stop flag is True - return 0 # return 0 to indicate that optimization operation is stopped - if hasattr(self.nn,'gradient'): # check if neural network model has a gradient attribute (custom gradient function) - gradient=self.nn.gradient(tape,loss) # compute the gradient using neural network model and loss - else: - if hasattr(self.nn,'nn'): # check if neural network model has a nn attribute (single network) - gradient=tape.gradient(loss,self.nn.param) # compute the gradient using loss and neural network parameters - else: # otherwise, assume neural network model does not have a nn attribute (actor-critic network) - actor_gradient=tape.gradient(loss[0],self.nn.param[0]) # compute the actor gradient using loss and actor parameters - critic_gradient=tape.gradient(loss[1],self.nn.param[1]) # compute the critic gradient using loss and critic parameters + if self.stop_flag.value==True: # if stop flag is True + return None,None # return None values for loss and parameter + if p==self.priority_p.value: # if process index is equal to priority process index + break # break out of the loop + else: # otherwise + continue # continue looping + lock[0].acquire() # acquire lock with index 0 + if self.stop_func_(lock[0]): # call stop_func_ method with lock as argument and check if it returns True + return None,None # return None values for loss and parameter + if hasattr(self.nn,'gradient'): # if neural network model has an attribute called gradient + gradient=self.nn.gradient(tape,loss) # calculate gradient by calling gradient attribute of neural network model with tape and loss as arguments + else: # otherwise, assume that neural network model does not have an attribute called gradient + if hasattr(self.nn,'nn'): # if neural network model has an attribute called nn + gradient=tape.gradient(loss,self.nn.param) # calculate gradient by calling gradient method of tape with loss and neural network model parameter as arguments + else: # otherwise, assume that neither nn nor gradient attributes exist in neural network model + actor_gradient=tape.gradient(loss[0],self.nn.param[0]) # calculate actor gradient by calling gradient method of tape with actor loss and actor parameter as arguments + critic_gradient=tape.gradient(loss[1],self.nn.param[1]) # calculate critic gradient by calling gradient method of tape with critic loss and critic parameter as arguments try: - if hasattr(self.nn,'attenuate'): # check if neural network model has an attenuate attribute (custom attenuate function) + if hasattr(self.nn,'attenuate'): # if neural network model has an attribute called attenuate try: - gradient=self.nn.attenuate(gradient,self.nn.opt_counter,p) # attenuate the gradient using neural network model, optimization counter, and process index + gradient=self.nn.attenuate(gradient,self.nn.opt_counter,p) # attenuate gradient by calling attenuate attribute of neural network model with gradient, optimization counter and process index as arguments except Exception: - actor_gradient=self.nn.attenuate(actor_gradient,self.nn.opt_counter,p) # attenuate the actor gradient using neural network model, optimization counter, and process index - critic_gradient=self.nn.attenuate(critic_gradient,self.nn.opt_counter,p) # attenuate the critic gradient using neural network model, optimization counter, and process index + actor_gradient=self.nn.attenuate(actor_gradient,self.nn.opt_counter,p) # if exception occurs, try to attenuate actor gradient by calling attenuate attribute of neural network model with actor gradient, optimization counter and process index as arguments + critic_gradient=self.nn.attenuate(critic_gradient,self.nn.opt_counter,p) # and try to attenuate critic gradient by calling attenuate attribute of neural network model with critic gradient, optimization counter and process index as arguments except Exception as e: - raise e # raise any exception + raise e # if exception still occurs, raise it try: try: - param=self.nn.opt(gradient) # update the neural network parameters using optimizer and gradient + param=self.nn.opt(gradient,p) # optimize parameter by calling opt attribute of neural network model with gradient and process index as arguments except Exception: - param=self.nn.opt(gradient,p) # update the neural network parameters using optimizer, gradient, and process index + param=self.nn.opt(gradient) # if exception occurs, try again without process index as argument except Exception as e: - raise e # raise any exception - lock[0].release() # release the lock for optimization operation - elif self.PO==2: # check if parallel optimization mode is 2 - g_lock.acquire() # acquire the global lock for gradient computation - if self.stop_func_(g_lock): # check if stop flag is True - return 0 # return 0 to indicate that optimization operation is stopped - if hasattr(self.nn,'gradient'): # check if neural network model has a gradient attribute (custom gradient function) - gradient=self.nn.gradient(tape,loss) # compute the gradient using neural network model and loss - else: - if hasattr(self.nn,'nn'): # check if neural network model has a nn attribute (single network) - gradient=tape.gradient(loss,self.nn.param) # compute the gradient using loss and neural network parameters - else: # otherwise, assume neural network model does not have a nn attribute (actor-critic network) - actor_gradient=tape.gradient(loss[0],self.nn.param[0]) # compute the actor gradient using loss and actor parameters - critic_gradient=tape.gradient(loss[1],self.nn.param[1]) # compute the critic gradient using loss and critic parameters - g_lock.release() # release the global lock for gradient computation - if self.priority_flag==True and self.priority_p.value!=-1: # check if priority flag is True and priority index is not -1 + raise e # if exception still occurs, raise it + lock[0].release() # release lock with index 0 + elif self.PO==2: # if parallel optimization mode is 2 + g_lock.acquire() # acquire global lock + if self.stop_func_(g_lock): # call stop_func_ method with global lock as argument and check if it returns True + return None,None # return None values for loss and parameter + if hasattr(self.nn,'gradient'): # if neural network model has an attribute called gradient + gradient=self.nn.gradient(tape,loss) # calculate gradient by calling gradient attribute of neural network model with tape and loss as arguments + else: # otherwise, assume that neural network model does not have an attribute called gradient + if hasattr(self.nn,'nn'): # if neural network model has an attribute called nn + gradient=tape.gradient(loss,self.nn.param) # calculate gradient by calling gradient method of tape with loss and neural network model parameter as arguments + else: # otherwise, assume that neither nn nor gradient attributes exist in neural network model + actor_gradient=tape.gradient(loss[0],self.nn.param[0]) # calculate actor gradient by calling gradient method of tape with actor loss and actor parameter as arguments + critic_gradient=tape.gradient(loss[1],self.nn.param[1]) # calculate critic gradient by calling gradient method of tape with critic loss and critic parameter as arguments + g_lock.release() # release global lock + if self.priority_flag==True and self.priority_p.value!=-1: # if priority optimization is enabled and priority process index is not -1 while True: - if p==self.priority_p.value: # check if current process index is equal to priority index - break # break the loop - else: - continue # try again - lock[0].acquire() # acquire the lock for optimization operation - if self.stop_func_(lock[0]): # check if stop flag is True - return 0 # return 0 to indicate that optimization operation is stopped + if self.stop_flag.value==True: # if stop flag is True + return None,None # return None values for loss and parameter + if p==self.priority_p.value: # if process index is equal to priority process index + break # break out of the loop + else: # otherwise + continue # continue looping + lock[0].acquire() # acquire lock with index 0 try: - if hasattr(self.nn,'attenuate'): # check if neural network model has an attenuate attribute (custom attenuate function) + if hasattr(self.nn,'attenuate'): # if neural network model has an attribute called attenuate try: - gradient=self.nn.attenuate(gradient,self.nn.opt_counter,p) # attenuate the gradient using neural network model, optimization counter, and process index + gradient=self.nn.attenuate(gradient,self.nn.opt_counter,p) # attenuate gradient by calling attenuate attribute of neural network model with gradient, optimization counter and process index as arguments except Exception: - actor_gradient=self.nn.attenuate(actor_gradient,self.nn.opt_counter,p) # attenuate the actor gradient using neural network model, optimization counter, and process index - critic_gradient=self.nn.attenuate(critic_gradient,self.nn.opt_counter,p) # attenuate the critic gradient using neural network model, optimization counter, and process index + actor_gradient=self.nn.attenuate(actor_gradient,self.nn.opt_counter,p) # if exception occurs, try to attenuate actor gradient by calling attenuate attribute of neural network model with actor gradient, optimization counter and process index as arguments + critic_gradient=self.nn.attenuate(critic_gradient,self.nn.opt_counter,p) # and try to attenuate critic gradient by calling attenuate attribute of neural network model with critic gradient, optimization counter and process index as arguments except Exception as e: - raise e # raise any exception + raise e # if exception still occurs, raise it try: try: - param=self.nn.opt(gradient) # update the neural network parameters using optimizer and gradient + param=self.nn.opt(gradient,p) # optimize parameter by calling opt attribute of neural network model with gradient and process index as arguments except Exception: - param=self.nn.opt(gradient,p) # update the neural network parameters using optimizer, gradient, and process index + param=self.nn.opt(gradient) # if exception occurs, try again without process index as argument except Exception as e: - raise e # raise any exception - lock[0].release() # release the lock for optimization operation - elif self.PO==3: # check if parallel optimization mode is 3 - if self.priority_flag==True and self.priority_p.value!=-1: # check if priority flag is True and priority index is not -1 + raise e # if exception still occurs, raise it + lock[0].release() # release lock with index 0 + elif self.PO==3: # if parallel optimization mode is 3 + if self.priority_flag==True and self.priority_p.value!=-1: # if priority optimization is enabled and priority process index is not -1 while True: - if p==self.priority_p.value: # check if current process index is equal to priority index - break # break the loop - else: - continue # try again - if self.stop_func_(): # check if stop flag is True - return 0 # return 0 to indicate that optimization operation is stopped - if hasattr(self.nn,'gradient'): # check if neural network model has a gradient attribute (custom gradient function) - gradient=self.nn.gradient(tape,loss) # compute the gradient using neural network model and loss - else: - if hasattr(self.nn,'nn'): # check if neural network model has a nn attribute (single network) - gradient=tape.gradient(loss,self.nn.param) # compute the gradient using loss and neural network parameters - else: # otherwise, assume neural network model does not have a nn attribute (actor-critic network) - actor_gradient=tape.gradient(loss[0],self.nn.param[0]) # compute the actor gradient using loss and actor parameters - critic_gradient=tape.gradient(loss[1],self.nn.param[1]) # compute the critic gradient using loss and critic parameters + if self.stop_flag.value==True: # if stop flag is True + return None,None # return None values for loss and parameter + if p==self.priority_p.value: # if process index is equal to priority process index + break # break out of the loop + else: # otherwise + continue # continue looping + if self.stop_func_(): # call stop_func_ method without lock as argument and check if it returns True + return None,None # return None values for loss and parameter + if hasattr(self.nn,'gradient'): # if neural network model has an attribute called gradient + gradient=self.nn.gradient(tape,loss) # calculate gradient by calling gradient attribute of neural network model with tape and loss as arguments + else: # otherwise, assume that neural network model does not have an attribute called gradient + if hasattr(self.nn,'nn'): # if neural network model has an attribute called nn + gradient=tape.gradient(loss,self.nn.param) # calculate gradient by calling gradient method of tape with loss and neural network model parameter as arguments + else: # otherwise, assume that neither nn nor gradient attributes exist in neural network model + actor_gradient=tape.gradient(loss[0],self.nn.param[0]) # calculate actor gradient by calling gradient method of tape with actor loss and actor parameter as arguments + critic_gradient=tape.gradient(loss[1],self.nn.param[1]) # calculate critic gradient by calling gradient method of tape with critic loss and critic parameter as arguments try: - if hasattr(self.nn,'attenuate'): # check if neural network model has an attenuate attribute (custom attenuate function) + if hasattr(self.nn,'attenuate'): # if neural network model has an attribute called attenuate try: - gradient=self.nn.attenuate(gradient,self.nn.opt_counter,p) # attenuate the gradient using neural network model, optimization counter, and process index + gradient=self.nn.attenuate(gradient,self.nn.opt_counter,p) # attenuate gradient by calling attenuate attribute of neural network model with gradient, optimization counter and process index as arguments except Exception: - actor_gradient=self.nn.attenuate(actor_gradient,self.nn.opt_counter,p) # attenuate the actor gradient using neural network model, optimization counter, and process index - critic_gradient=self.nn.attenuate(critic_gradient,self.nn.opt_counter,p) # attenuate the critic gradient using neural network model, optimization counter, and process index + actor_gradient=self.nn.attenuate(actor_gradient,self.nn.opt_counter,p) # if exception occurs, try to attenuate actor gradient by calling attenuate attribute of neural network model with actor gradient, optimization counter and process index as arguments + critic_gradient=self.nn.attenuate(critic_gradient,self.nn.opt_counter,p) # and try to attenuate critic gradient by calling attenuate attribute of neural network model with critic gradient, optimization counter and process index as arguments except Exception as e: - raise e # raise any exception + raise e # if exception still occurs, raise it try: try: - param=self.nn.opt(gradient) # update the neural network parameters using optimizer and gradient + param=self.nn.opt(gradient,p) # optimize parameter by calling opt attribute of neural network model with gradient and process index as arguments except Exception: - param=self.nn.opt(gradient,p) # update the neural network parameters using optimizer, gradient, and process index + param=self.nn.opt(gradient) # if exception occurs, try again without process index as argument except Exception as e: - raise e # raise any exception - return loss,param # return the loss and the updated parameters + raise e # if exception still occurs, raise it + return loss,param # return loss and parameter def update_nn_param(self,param=None): - if param==None: # check if param is None - parameter_flat=nest.flatten(self.nn.param) # flatten the neural network parameters into a list - parameter7_flat=nest.flatten(self.param[7]) # flatten the dictionary parameters into a list - else: # otherwise, assume param is not None - parameter_flat=nest.flatten(self.nn.param) # flatten the neural network parameters into a list - parameter7_flat=nest.flatten(param) # flatten the param into a list - for i in range(len(parameter_flat)): # loop over each element in the lists - if param==None: # check if param is None - state_ops.assign(parameter_flat[i],parameter7_flat[i]) # assign the value of dictionary parameter to neural network parameter - else: # otherwise, assume param is not None - state_ops.assign(parameter_flat[i],parameter7_flat[i]) # assign the value of param to neural network parameter - self.nn.param=nest.pack_sequence_as(self.nn.param,parameter_flat) # pack the list back into the original structure of neural network parameters - self.param[7]=nest.pack_sequence_as(self.param[7],parameter7_flat) # pack the list back into the original structure of dictionary parameters + if param==None: # if parameter is not given + parameter_flat=nest.flatten(self.nn.param) # flatten the neural network model parameter to a list + parameter7_flat=nest.flatten(self.param[7]) # flatten the parameter with key 7 in the shared dictionary to a list + else: # otherwise, assume that parameter is given + parameter_flat=nest.flatten(self.nn.param) # flatten the neural network model parameter to a list + parameter7_flat=nest.flatten(param) # flatten the given parameter to a list + for i in range(len(parameter_flat)): # loop through the indices of the flattened lists + if param==None: # if parameter is not given + state_ops.assign(parameter_flat[i],parameter7_flat[i]) # assign the value of the parameter with key 7 in the shared dictionary to the neural network model parameter + else: # otherwise, assume that parameter is given + state_ops.assign(parameter_flat[i],parameter7_flat[i]) # assign the value of the given parameter to the neural network model parameter + self.nn.param=nest.pack_sequence_as(self.nn.param,parameter_flat) # pack the flattened list back to the original structure of the neural network model parameter + self.param[7]=nest.pack_sequence_as(self.param[7],parameter7_flat) # pack the flattened list back to the original structure of the parameter with key 7 in the shared dictionary return def _train(self,p,j,batches,length,lock,g_lock): - if j==batches-1: # check if this is the last batch of data - index1=batches*self.batch # get the start index of the batch from the pool - index2=self.batch-(length-batches*self.batch) # get the end index of the batch from the pool - state_batch=np.concatenate((self.state_pool[p][index1:length],self.state_pool[p][:index2]),0) # concatenate the state data from two parts of the pool - action_batch=np.concatenate((self.action_pool[p][index1:length],self.action_pool[p][:index2]),0) # concatenate the action data from two parts of the pool - next_state_batch=np.concatenate((self.next_state_pool[p][index1:length],self.next_state_pool[p][:index2]),0) # concatenate the next state data from two parts of the pool - reward_batch=np.concatenate((self.reward_pool[p][index1:length],self.reward_pool[p][:index2]),0) # concatenate the reward data from two parts of the pool - done_batch=np.concatenate((self.done_pool[p][index1:length],self.done_pool[p][:index2]),0) # concatenate the done flag data from two parts of the pool - if self.PO==2: # check if parallel optimization mode is 2 - if type(g_lock)!=list: # check if g_lock is not a list - pass # do nothing - elif len(g_lock)==self.process: # check if g_lock has the same length as the number of processes - ln=p # assign the current process index to ln - g_lock=g_lock[ln] # assign the corresponding lock to g_lock - else: # otherwise, assume g_lock has a different length from the number of processes - ln=int(np.random.choice(len(g_lock))) # randomly choose an index from g_lock - g_lock=g_lock[ln] # assign the corresponding lock to g_lock - loss,param=self.opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch,p,lock,g_lock) # execute the optimization operation using the batch data, process index, lock, and g_lock - else: # otherwise, assume parallel optimization mode is not 2 - loss,param=self.opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch,p,lock) # execute the optimization operation using the batch data, process index, and lock - self.param[7]=param # assign the updated parameters to the dictionary - self.loss[p]+=loss # add the loss to the loss array of the current process - if hasattr(self.nn,'bc'): # check if neural network model has a bc attribute (batch counter) - bc=self.nn.bc[0] # get the batch counter from neural network model - bc.assign_add(1) # increment the batch counter by 1 - self.nn.bc[0]=bc # assign the batch counter back to neural network model - else: # otherwise, assume this is not the last batch of data - index1=j*self.batch # get the start index of the batch from the pool - index2=(j+1)*self.batch # get the end index of the batch from the pool - state_batch=self.state_pool[p][index1:index2] # get the state data from the pool - action_batch=self.action_pool[p][index1:index2] # get the action data from the pool - next_state_batch=self.next_state_pool[p][index1:index2] # get the next state data from the pool - reward_batch=self.reward_pool[p][index1:index2] # get the reward data from the pool - done_batch=self.done_pool[p][index1:index2] # get the done flag data from the pool - if self.PO==2: # check if parallel optimization mode is 2 - if type(g_lock)!=list: # check if g_lock is not a list - pass # do nothing - elif len(g_lock)==self.process: # check if g_lock has the same length as the number of processes - ln=p # assign the current process index to ln - g_lock=g_lock[ln] # assign the corresponding lock to g_lock - else: # otherwise, assume g_lock has a different length from the number of processes - ln=int(np.random.choice(len(g_lock))) # randomly choose an index from g_lock - g_lock=g_lock[ln] # assign the corresponding lock to g_lock - loss,param=self.opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch,p,lock,g_lock) # execute the optimization operation using the batch data, process index, lock, and g_lock - else: # otherwise, assume parallel optimization mode is not 2 - loss,param=self.opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch,p,lock) # execute the optimization operation using the batch data, process index, and lock - self.param[7]=param # assign the updated parameters to the dictionary - self.loss[p]+=loss # add the loss to the loss array of the current process - if hasattr(self.nn,'bc'): # check if neural network model has a bc attribute (batch counter) - bc=self.nn.bc[0] # get the batch counter from neural network model - bc.assign_add(1) # increment the batch counter by 1 - self.nn.bc[0]=bc # assign the batch counter back to neural network model - return + if j==batches-1: # if it is the last batch + index1=batches*self.batch # calculate the start index of the batch + index2=self.batch-(length-batches*self.batch) # calculate the end index of the batch + state_batch=np.concatenate((self.state_pool[p][index1:length],self.state_pool[p][:index2]),0) # concatenate state data from two parts of the state pool to form a complete batch + action_batch=np.concatenate((self.action_pool[p][index1:length],self.action_pool[p][:index2]),0) # concatenate action data from two parts of the action pool to form a complete batch + next_state_batch=np.concatenate((self.next_state_pool[p][index1:length],self.next_state_pool[p][:index2]),0) # concatenate next state data from two parts of the next state pool to form a complete batch + reward_batch=np.concatenate((self.reward_pool[p][index1:length],self.reward_pool[p][:index2]),0) # concatenate reward data from two parts of the reward pool to form a complete batch + done_batch=np.concatenate((self.done_pool[p][index1:length],self.done_pool[p][:index2]),0) # concatenate done flag data from two parts of the done pool to form a complete batch + if self.PO==2: # if parallel optimization mode is 2 + if type(g_lock)!=list: # if global lock is not a list + pass # do nothing + elif len(g_lock)==self.process: # if global lock is a list with length equal to process count + ln=p # set local lock index to process index + g_lock=g_lock[ln] # set global lock to local lock with index ln + else: # otherwise, assume that global lock is a list with length not equal to process count + ln=int(np.random.choice(len(g_lock))) # randomly choose a local lock index from global lock list + g_lock=g_lock[ln] # set global lock to local lock with index ln + loss,param=self.opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch,p,lock,g_lock) # call opt method with batch data, process index, lock and global lock as arguments and get loss and parameter as outputs + else: # otherwise, assume that parallel optimization mode is not 2 + loss,param=self.opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch,p,lock) # call opt method with batch data, process index and lock as arguments and get loss and parameter as outputs + if self.stop_flag.value==True: # if stop flag is True + return # return without doing anything + self.param[7]=param # assign parameter to parameter with key 7 in shared dictionary + self.loss[p]+=loss # add loss to loss array for process index p + if hasattr(self.nn,'bc'): # if neural network model has an attribute called bc (batch counter) + bc=self.nn.bc[0] # get bc value from neural network model + bc.assign_add(1) # increment bc value by 1 + self.nn.bc[0]=bc # assign bc value back to neural network + else: # otherwise, assume that it is not the last batch + index1=j*self.batch # calculate the start index of the batch + index2=(j+1)*self.batch # calculate the end index of the batch + state_batch=self.state_pool[p][index1:index2] # get state data from state pool with start and end indices + action_batch=self.action_pool[p][index1:index2] # get action data from action pool with start and end indices + next_state_batch=self.next_state_pool[p][index1:index2] # get next state data from next state pool with start and end indices + reward_batch=self.reward_pool[p][index1:index2] # get reward data from reward pool with start and end indices + done_batch=self.done_pool[p][index1:index2] # get done flag data from done pool with start and end indices + if self.PO==2: # if parallel optimization mode is 2 + if type(g_lock)!=list: # if global lock is not a list + pass # do nothing + elif len(g_lock)==self.process: # if global lock is a list with length equal to process count + ln=p # set local lock index to process index + g_lock=g_lock[ln] # set global lock to local lock with index ln + else: # otherwise, assume that global lock is a list with length not equal to process count + ln=int(np.random.choice(len(g_lock))) # randomly choose a local lock index from global lock list + g_lock=g_lock[ln] # set global lock to local lock with index ln + loss,param=self.opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch,p,lock,g_lock) # call opt method with batch data, process index, lock and global lock as arguments and get loss and parameter as outputs + else: # otherwise, assume that parallel optimization mode is not 2 + loss,param=self.opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch,p,lock) # call opt method with batch data, process index and lock as arguments and get loss and parameter as outputs + if self.stop_flag.value==True: # if stop flag is True + return # return without doing anything + self.param[7]=param # assign parameter to parameter with key 7 in shared dictionary + self.loss[p]+=loss # add loss to loss array for process index p + if hasattr(self.nn,'bc'): # if neural network model has an attribute called bc (batch counter) + bc=self.nn.bc[0] # get bc value from neural network model + bc.assign_add(1) # increment bc value by 1 + self.nn.bc[0]=bc # assign bc value back to neural network model + return # return without any value def train_(self,p,lock,g_lock): - if len(self.done_pool[p])=self.max_opt: # check if maximum number of optimization steps per process per episode is set and reached - self.priority_p.value=int(self.priority_p.value) # convert the priority index to an integer - elif self.max_opt==None: # check if maximum number of optimization steps per process per episode is not set - self.priority_p.value=int(self.priority_p.value) # convert the priority index to an integer - else: # otherwise, assume maximum number of optimization steps per process per episode is set but not reached - self.priority_p.value=-1 # assign the priority index as -1 to indicate no priority - if self.priority_flag==True: # check if priority flag is True - self.opt_counter[p]=0 # reset the optimization counter for current process as zero - if hasattr(self.nn,'attenuate'): # check if neural network model has an attenuate attribute (custom attenuate function) - opt_counter=self.nn.opt_counter[0] # get the optimization counter from neural network model - opt_counter.scatter_update(tf.IndexedSlices(0,p)) # reset the optimization counter for current process as zero using scatter update operation - self.nn.opt_counter[0]=opt_counter # assign the optimization counter back to neural network model - self._train(p,j,batches,length,lock,g_lock) # execute the training operation using the batch data, process index, lock, and g_lock - if self.priority_flag==True: # check if priority flag is True - opt_counter=np.frombuffer(self.opt_counter.get_obj(),dtype='i') # get the optimization counter array from shared memory - opt_counter+=1 # increment the optimization counter array by 1 - if hasattr(self.nn,'attenuate'): # check if neural network model has an attenuate attribute (custom attenuate function) - opt_counter=self.nn.opt_counter[0] # get the optimization counter from neural network model - opt_counter.assign(opt_counter+1) # increment the optimization counter by 1 using assign operation - self.nn.opt_counter[0]=opt_counter # assign the optimization counter back to neural network model - if self.PO==1 or self.PO==2: # check if parallel optimization mode is 1 or 2 - lock[1].acquire() # acquire the lock for saving and updating operation - if self.update_step!=None: # check if update frequency for target network is set - if self.sc[p]%self.update_step==0: # check if step counter for current process reaches the update frequency - self.nn.update_param() # update the target network parameters using neural network model - else: # otherwise, assume update frequency for target network is not set - self.nn.update_param() # update the target network parameters using neural network model - if self.PO==1 or self.PO==2: # check if parallel optimization mode is 1 or 2 - lock[1].release() # release the lock for saving and updating operation - self.loss[p]=self.loss[p]/batches # compute the average loss for current process by dividing by batches - self.sc[p]+=1 # increment the step counter for current process by 1 - if hasattr(self.nn,'ec'): # check if neural network model has an ec attribute (episode counter) - ec=self.nn.ec[0] # get the episode counter from neural network model - ec.assign_add(1) # increment the episode counter by 1 using assign operation - self.nn.ec[0]=ec # assign the episode counter back to neural network model - return + if len(self.done_pool[p])=self.max_opt: # if maximum optimization step value is given and optimization counter value for priority process index is greater than or equal to it + self.priority_p.value=int(self.priority_p.value) # convert priority process index to integer type + elif self.max_opt==None: # otherwise, assume that maximum optimization step value is not given + self.priority_p.value=int(self.priority_p.value) # convert priority process index to integer type + else: # otherwise, assume that maximum optimization step value is given but optimization counter value for priority process index is less than it + self.priority_p.value=-1 # set priority process index to -1, meaning no priority process + if self.priority_flag==True: # if priority optimization is enabled + self.opt_counter[p]=0 # reset optimization counter value for process index p to zero + if hasattr(self.nn,'attenuate'): # if neural network model has an attribute called attenuate + opt_counter=self.nn.opt_counter[0] # get opt_counter value from neural network model + opt_counter.scatter_update(tf.IndexedSlices(0,p)) # update opt_counter value with 0 at process index p + self.nn.opt_counter[0]=opt_counter # assign opt_counter value back to neural network model + self._train(p,j,batches,length,lock,g_lock) # call _train method with process index, batch index, batch number, length, lock and global lock as arguments + if self.stop_flag.value==True: # if stop flag is True + return # return without doing anything + if self.priority_flag==True: # if priority optimization is enabled + opt_counter=np.frombuffer(self.opt_counter.get_obj(),dtype='i') # get optimization counter array from shared memory + opt_counter+=1 # increment optimization counter array by 1 + if hasattr(self.nn,'attenuate'): # if neural network model has an attribute called attenuate + opt_counter=self.nn.opt_counter[0] # get opt_counter value from neural network model + opt_counter.assign(opt_counter+1) # increment opt_counter value by 1 + self.nn.opt_counter[0]=opt_counter # assign opt_counter value back to neural network model + if self.PO==1 or self.PO==2: # if parallel optimization mode is 1 or 2 + lock[1].acquire() # acquire lock with index 1 + if self.update_step!=None: # if update step value is given + if self.sc[p]%self.update_step==0: # if step counter value for process index p is divisible by update step value + self.nn.update_param() # call update_param method of neural network model to update its parameters + else: # otherwise, assume that update step value is not given + self.nn.update_param() # call update_param method of neural network model to update its parameters + if self.PO==1 or self.PO==2: # if parallel optimization mode is 1 or 2 + lock[1].release() # release lock with index 1 + self.loss[p]=self.loss[p]/batches # calculate the average loss for process index p by dividing the total loss by the number of batches + self.sc[p]+=1 # increment the step counter value for process index p by 1 + if hasattr(self.nn,'ec'): # if neural network model has an attribute called ec (episode counter) + ec=self.nn.ec[0] # get ec value from neural network model + ec.assign_add(1) # increment ec value by 1 + self.nn.ec[0]=ec # assign ec value back to neural network model + return # return without any value - def train(self,p,episode_count,lock,pool_lock,g_lock=None): - if self.PO==1 or self.PO==2: # check if parallel optimization mode is 1 or 2 - lock[1].acquire() # acquire the lock for initialization operation - elif self.PO==3: # check if parallel optimization mode is 3 - lock[1].acquire() # acquire the lock for initialization operation - self.state_pool[p]=None # initialize the state pool for current process as None - self.action_pool[p]=None # initialize the action pool for current process as None - self.next_state_pool[p]=None # initialize the next state pool for current process as None - self.reward_pool[p]=None # initialize the reward pool for current process as None - self.done_pool[p]=None # initialize the done pool for current process as None - self.running_flag.append(1) # append a running flag of 1 to indicate that current process is running - self.process_counter.value+=1 # increment the process counter by 1 - self.finish_list.append(None) # append a finish flag of None to indicate that current process is not finished - if self.PO==1 or self.PO==2: # check if parallel optimization mode is 1 or 2 - lock[1].release() # release the lock for initialization operation - elif self.PO==3: # check if parallel optimization mode is 3 - lock[1].release() # release the lock for initialization operation + def train(self,p,lock,pool_lock,g_lock=None): + if self.PO==1 or self.PO==2: # if parallel optimization mode is 1 or 2 + lock[1].acquire() # acquire lock with index 1 + elif self.PO==3: # if parallel optimization mode is 3 + lock[1].acquire() # acquire lock with index 1 + self.state_pool[p]=None # set state pool for process index p to None + self.action_pool[p]=None # set action pool for process index p to None + self.next_state_pool[p]=None # set next state pool for process index p to None + self.reward_pool[p]=None # set reward pool for process index p to None + self.done_pool[p]=None # set done pool for process index p to None + self.running_flag.append(1) # append 1 to running flag list, indicating that process is running + self.process_counter.value+=1 # increment process counter value by 1, indicating that one more process is running + self.finish_list.append(None) # append None to finish list, indicating that process is not finished yet + if self.PO==1 or self.PO==2: # if parallel optimization mode is 1 or 2 + lock[1].release() # release lock with index 1 + elif self.PO==3: # if parallel optimization mode is 3 + lock[1].release() # release lock with index 1 try: - epsilon=self.epsilon[p] # get the exploration rate for current process + epsilon=self.epsilon[p] # get epsilon value for process index p from epsilon array except Exception: - epsilon=None # set the exploration rate as None if exception occurs - for k in range(episode_count): # loop over each episode - s=self.nn.env(p=p,initial=True) # reset the environment and get the initial state - if type(self.nn.param[0])!=list: - s=np.array(s,self.nn.param[0].dtype.name) # convert the state to an array with the same data type as neural network parameters - else: - s=np.array(s,self.nn.param[0][0].dtype.name) # convert the state to an array with the same data type as neural network parameters - if self.episode_step==None: # check if maximum number of steps per episode is not set - while True: # loop until episode ends - next_s,r,done,index=self.env(s,epsilon,p,lock,pool_lock) # execute an action and get the next state, reward, done flag, and index - self.reward[p]+=r # add the reward to the reward array of current process - s=next_s # update the state as next state - if type(self.done_pool[p])==np.ndarray: # check if done pool is an array (enough data for training) - self.train_(p,lock,g_lock) # execute the training operation using current process index, lock, and g_lock - if done: # check if episode ends - if self.PO==1 or self.PO==2: # check if parallel optimization mode is 1 or 2 - lock[1].acquire() # acquire the lock for episode operation - self.total_episode.value+=1 # increment the total episode counter by 1 - self.loss_list.append(self.loss[p]) # append the loss of current process to the loss list - if self.PO==1 or self.PO==2: # check if parallel optimization mode is 1 or 2 - lock[1].release() # release the lock for episode operation - break # break the loop - else: # otherwise, assume maximum number of steps per episode is set - for l in range(self.episode_step): # loop over each step - next_s,r,done,index=self.env(s,epsilon,p,lock,pool_lock) # execute an action and get the next state, reward, done flag, and index - self.reward[p]+=r # add the reward to the reward array of current process - s=next_s # update the state as next state - if type(self.done_pool[p])==np.ndarray: # check if done pool is an array (enough data for training) - self.train_(p,lock,g_lock) # execute the training operation using current process index, lock, and g_lock - if done: # check if episode ends - if self.PO==1 or self.PO==2: # check if parallel optimization mode is 1 or 2 - lock[1].acquire() # acquire the lock for episode operation - self.total_episode.value+=1 # increment the total episode counter by 1 - self.loss_list.append(self.loss[p]) # append the loss of current process to the loss list - if self.PO==1 or self.PO==2: # check if parallel optimization mode is 1 or 2 - lock[1].release() # release the lock for episode operation - break # break the loop - if l==self.episode_step-1: # check if this is the last step of episode - if self.PO==1 or self.PO==2: # check if parallel optimization mode is 1 or 2 - lock[1].acquire() # acquire the lock for episode operation - self.total_episode.value+=1 # increment the total episode counter by 1 - self.loss_list.append(self.loss[p]) # append the loss of current process to the loss list - if self.PO==1 or self.PO==2: # check if parallel optimization mode is 1 or 2 - lock[1].release() # release the lock for episode operation - if self.PO==1 or self.PO==2: # check if parallel optimization mode is 1 or 2 - lock[1].acquire() # acquire the lock for saving and updating operation - elif len(lock)==3: # check if there are three locks (for saving and updating operation) - lock[2].acquire() # acquire the third lock for saving and updating operation - if self.update_step!=None: # check if update frequency for target network is set - if self.sc[p]%self.update_step==0: # check if step counter for current process reaches the update frequency - self.nn.update_param() # update the target network parameters using neural network model - else: # otherwise, assume update frequency for target network is not set - self.nn.update_param() # update the target network parameters using neural network model - if self.PO==1 or self.PO==2: # check if parallel optimization mode is 1 or 2 - lock[1].release() # release the lock for saving and updating operation - elif len(lock)==3: # check if there are three locks (for saving and updating operation) - lock[2].release() # release the third lock for saving and updating operation - if self.PO==1 or self.PO==2: # check if parallel optimization mode is 1 or 2 - lock[1].acquire() # acquire the lock for saving and updating operation - elif len(lock)==3: # check if there are three locks (for saving and updating operation) - lock[2].acquire() # acquire the third lock for saving and updating operation - self.save_() # execute the save_ function to save - self.reward_list.append(self.reward[p]) # append the reward of current process to the reward list - self.reward[p]=0 # reset the reward of current process as zero - if self.PO==1 or self.PO==2: # check if parallel optimization mode is 1 or 2 - lock[1].release() # release the lock for saving and updating operation - elif len(lock)==3: # check if there are three locks (for saving and updating operation) - lock[2].release() # release the third lock for saving and updating operation - self.running_flag[p+1]=0 # set the running flag of current process as zero to indicate that it is not running anymore - if p not in self.finish_list: # check if current process is not finished yet - self.finish_list[p]=p # assign the process index to the finish list to indicate that it is finished - if self.PO==1 or self.PO==2: # check if parallel optimization mode is 1 or 2 - lock[1].acquire() # acquire the lock for episode operation - elif self.PO==3: # check if parallel optimization mode is 3 - lock[1].acquire() # acquire the lock for episode operation - self.process_counter.value-=1 # decrement the process counter by 1 - if self.PO==1 or self.PO==2: # check if parallel optimization mode is 1 or 2 - lock[1].release() # release the lock for episode operation - elif self.PO==3: # check if parallel optimization mode is 3 - lock[1].release() # release the lock for episode operation - del self.state_pool[p] # delete the state pool of current process - del self.action_pool[p] # delete the action pool of current process - del self.next_state_pool[p] # delete the next state pool of current process - del self.reward_pool[p] # delete the reward pool of current process - del self.done_pool[p] # delete the done pool of current process - return # return to indicate that end operation is done + epsilon=None # if exception occurs, set epsilon value to None + while True: # loop indefinitely until break condition is met + if self.stop_flag.value==True: # if stop flag is True + break # break out of the loop + if self.episode_counter.value>=self.episode: # if episode counter value is greater than or equal to episode + break # break out of the loop + s=self.nn.env(p=p,initial=True) # get initial state from calling env attribute of neural network model with process index and initial flag as arguments + if type(self.nn.param[0])!=list: # if neural network model parameter is not a list + s=np.array(s,self.nn.param[0].dtype.name) # convert state data to the same data type as the neural network model parameter + else: # otherwise, assume that neural network model parameter is a list + s=np.array(s,self.nn.param[0][0].dtype.name) # convert state data to the same data type as the first element of the neural network model parameter list + if self.episode_step==None: # if episode step value is not given + while True: # loop indefinitely until break condition is met + next_s,r,done,index=self.env(s,epsilon,p,lock,pool_lock) # get next state, reward, done flag and index from calling env method with state, epsilon, process index, lock and pool lock as arguments + self.reward[p]+=r # add reward to reward array for process index p + s=next_s # set state to next state + if type(self.done_pool[p])==np.ndarray: # if done pool for process index p is a numpy array + self.train_(p,lock,g_lock) # call train_ method with process index, lock and global lock as arguments + if self.stop_flag.value==True: # if stop flag is True + break # break out of the loop + if done: # if done flag is True + if self.PO==1 or self.PO==2: # if parallel optimization mode is 1 or 2 + lock[1].acquire() # acquire lock with index 1 + elif len(lock)==4: # otherwise, if lock has length 4 + lock[3].acquire() # acquire lock with index 3 + self.episode_counter.value+=1 # increment episode counter value by 1, indicating that one more episode is completed + self.total_episode.value+=1 # increment total episode value by 1, indicating that one more episode is completed + self.loss_list.append(self.loss[p]) # append loss value for process index p to loss list + if self.PO==1 or self.PO==2: # if parallel optimization mode is 1 or 2 + lock[1].release() # release lock with index 1 + elif len(lock)==4: # otherwise, if lock has length 4 + lock[3].release() # release lock with index 3 + break # break out of the loop + else: # otherwise, assume that episode step value is given + for l in range(self.episode_step): # loop through the episode step indices + if self.episode_counter.value>=self.episode: # if episode counter value is greater than or equal to episode + break # break out of the loop + next_s,r,done,index=self.env(s,epsilon,p,lock,pool_lock) # get next state, reward, done flag and index from calling env method with state, epsilon, process index, lock and pool lock as arguments + self.reward[p]+=r # add reward to reward array for process index p + s=next_s # set state to next state + if type(self.done_pool[p])==np.ndarray: # if done pool for process index p is a numpy array + self.train_(p,lock,g_lock) # call train_ method with process index, lock and global lock as arguments + if self.stop_flag.value==True: # if stop flag is True + break # break out of the loop + if done: # if done flag is True + if self.PO==1 or self.PO==2: # if parallel optimization mode is 1 or 2 + lock[1].acquire() # acquire lock with index 1 + elif len(lock)==4: # otherwise, if lock has length 4 + lock[3].acquire() # acquire lock with index 3 + self.episode_counter.value+=1 # increment episode counter value by 1, indicating that one more episode is completed + self.total_episode.value+=1 # increment total episode value by 1, indicating that one more episode is completed + self.loss_list.append(self.loss[p]) # append loss value for process index p to loss list + if self.PO==1 or self.PO==2: # if parallel optimization mode is 1 or 2 + lock[1].release() # release lock with index 1 + elif len(lock)==4: # otherwise, if lock has length 4 + lock[3].release() # release lock with index 3 + break # break out of the loop + if l==self.episode_step-1: # if it is the last episode step + if self.PO==1 or self.PO==2: # if parallel optimization mode is 1 or 2 + lock[1].acquire() # acquire lock with index 1 + elif len(lock)==4: # otherwise, if lock has length 4 + lock[3].acquire() # acquire lock with index 3 + self.episode_counter.value+=1 # increment episode counter value by 1, indicating that one more episode is completed + self.total_episode.value+=1 # increment total episode value by 1, indicating that one more episode is completed + self.loss_list.append(self.loss[p]) # append loss value for process index p to loss list + if self.PO==1 or self.PO==2: # if parallel optimization mode is 1 or 2 + lock[1].release() # release lock with index 1 + elif len(lock)==4: # otherwise, if lock has length 4 + lock[3].release() # release lock with index 3 + if self.PO==1 or self.PO==2: # if parallel optimization mode is 1 or 2 + lock[1].acquire() # acquire lock with index 1 + elif len(lock)==3 or len(lock)==4: # otherwise, if lock has length 3 or 4 + lock[2].acquire() # acquire lock with index 2 + self.save_() # call save_ method to save neural network parameters to file + self.reward_list.append(self.reward[p]) # append reward value for process index p to reward list + self.reward[p]=0 # reset reward value for process index p to zero + if self.PO==1 or self.PO==2: # if parallel optimization mode is 1 or 2 + lock[1].release() # release lock with index 1 + elif len(lock)==3 or len(lock)==4: # otherwise, if lock has length 3 or 4 + lock[2].release() # release lock with index 2 + self.running_flag[p+1]=0 # set running flag value for process index p plus one to zero, indicating that process is not running anymore + if p not in self.finish_list: # if process index p is not in finish list + self.finish_list[p]=p # assign process index p to finish list at process index p, indicating that process is finished + if self.PO==1 or self.PO==2: # if parallel optimization mode is 1 or 2 + lock[1].acquire() # acquire lock with index 1 + elif self.PO==3: # if parallel optimization mode is 3 + lock[1].acquire() # acquire lock with index 1 + self.process_counter.value-=1 # decrement process counter value by 1, indicating that one less process is running + if self.PO==1 or self.PO==2: # if parallel optimization mode is 1 or 2 + lock[1].release() # release lock with index 1 + elif self.PO==3: # if parallel optimization mode is 3 + lock[1].release() # release lock with index 1 + del self.state_pool[p] # delete state pool for process index p + del self.action_pool[p] # delete action pool for process index p + del self.next_state_pool[p] # delete next state pool for process index p + del self.reward_pool[p] # delete reward pool for process index p + del self.done_pool[p] # delete done pool for process index p + return # return without any value def train_online(self,p,lock=None,g_lock=None): - if hasattr(self.nn,'counter'): # check if neural network model has a counter attribute (training counter) - self.nn.counter.append(0) # append a zero to indicate that current process has not started training yet - while True: # loop until training stops - if hasattr(self.nn,'save'): # check if neural network model has a save attribute (custom save function) - self.nn.save(self.save,p) # save the neural network parameters using neural network model and process index - if hasattr(self.nn,'stop_flag'): # check if neural network model has a stop_flag attribute - if self.nn.stop_flag==True: # check if stop flag is True - return # return to indicate that training stops - if hasattr(self.nn,'stop_func'): # check if neural network model has a stop_func attribute (custom stop function) - if self.nn.stop_func(p): # check if stop function returns True for current process index - return # return to indicate that training stops - if hasattr(self.nn,'suspend_func'): # check if neural network model has a suspend_func attribute (custom suspend function) - self.nn.suspend_func(p) # execute the suspend function for current process index + if hasattr(self.nn,'counter'): # if neural network model has an attribute called counter + self.nn.counter.append(0) # append zero to counter list of neural network model + while True: # loop indefinitely until break condition is met + if hasattr(self.nn,'save'): # if neural network model has an attribute called save + self.nn.save(self.save,p) # call save attribute of neural network model with save and process index as arguments + if hasattr(self.nn,'stop_flag'): # if neural network model has an attribute called stop_flag + if self.nn.stop_flag==True: # if stop_flag value of neural network model is True + return # return without doing anything + if hasattr(self.nn,'stop_func'): # if neural network model has an attribute called stop_func + if self.nn.stop_func(p): # call stop_func attribute of neural network model with process index as argument and check if it returns True + return # return without doing anything + if hasattr(self.nn,'suspend_func'): # if neural network model has an attribute called suspend_func + self.nn.suspend_func(p) # call suspend_func attribute of neural network model with process index as argument try: - data=self.nn.online(p) # get the data for online training using neural network model and process index + data=self.nn.online(p) # get data from calling online attribute of neural network model with process index as argument except Exception as e: - self.nn.exception_list[p]=e # store the exception to the exception list of neural network model - if data=='stop': # check if data is 'stop' - return # return to indicate that training stops - elif data=='suspend': # check if data is 'suspend' - self.nn.suspend_func(p) # execute the suspend function for current process index + self.nn.exception_list[p]=e # if exception occurs, assign it to exception list of neural network model at process index p + if data=='stop': # if data is 'stop' + return # return without doing anything + elif data=='suspend': # if data is 'suspend' + self.nn.suspend_func(p) # call suspend_func attribute of neural network model with process index as argument try: - if self.PO==2: # check if parallel optimization mode is 2 - if type(g_lock)!=list: # check if g_lock is not a list - pass # do nothing - elif len(g_lock)==self.process: # check if g_lock has the same length as the number of processes - ln=p # assign the current process index to ln - g_lock=g_lock[ln] # assign the corresponding lock to g_lock - else: # otherwise, assume g_lock has a different length from the number of processes - ln=int(np.random.choice(len(g_lock))) # randomly choose an index from g_lock - g_lock=g_lock[ln] # assign the corresponding lock to g_lock - loss,param=self.opt(data[0],data[1],data[2],data[3],data[4],p,lock,g_lock) # execute the optimization operation using the data, process index, lock, and g_lock - else: # otherwise, assume parallel optimization mode is not 2 - loss,param=self.opt(data[0],data[1],data[2],data[3],data[4],p,lock) # execute the optimization operation using the data, process index, and lock - self.param[7]=param # assign the updated parameters to the dictionary + if self.PO==2: # if parallel optimization mode is 2 + if type(g_lock)!=list: # if global lock is not a list + pass # do nothing + elif len(g_lock)==self.process: # if global lock is a list with length equal to process count + ln=p # set local lock index to process index + g_lock=g_lock[ln] # set global lock to local lock with index ln + else: # otherwise, assume that global lock is a list with length not equal to process count + ln=int(np.random.choice(len(g_lock))) # randomly choose a local lock index from global lock list + g_lock=g_lock[ln] # set global lock to local lock with index ln + loss,param=self.opt(data[0],data[1],data[2],data[3],data[4],p,lock,g_lock) # call opt method with data, process index, lock and global lock as arguments and get loss and parameter as outputs + else: # otherwise, assume that parallel optimization mode is not 2 + loss,param=self.opt(data[0],data[1],data[2],data[3],data[4],p,lock) # call opt method with data, process index and lock as arguments and get loss and parameter as outputs except Exception as e: - if self.PO==1: # check if parallel optimization mode is 1 - if lock[0].acquire(False): # try to acquire the lock for optimization operation - lock[0].release() # release the lock for optimization operation - elif self.PO==2: # check if parallel optimization mode is 2 - if g_lock.acquire(False): # try to acquire the global lock for gradient computation - g_lock.release() # release the global lock for gradient computation - if lock[0].acquire(False): # try to acquire the lock for optimization operation - lock[0].release() # release the lock for optimization operation - self.nn.exception_list[p]=e # store the exception to the exception list of neural network model - loss=loss.numpy() # convert the loss to a numpy array - if len(self.nn.train_loss_list)==self.nn.max_length: # check if train loss list reaches the maximum length - del self.nn.train_loss_list[0] # delete the oldest train loss from the list - self.nn.train_loss_list.append(loss) # append the train loss to the list + if self.PO==1: # if parallel optimization mode is 1 + if lock[0].acquire(False): # try to acquire lock with index 0 without blocking + lock[0].release() # release lock with index 0 + elif self.PO==2: # if parallel optimization mode is 2 + if g_lock.acquire(False): # try to acquire global lock without blocking + g_lock.release() # release global lock + if lock[0].acquire(False): # try to acquire lock with index 0 without blocking + lock[0].release() # release lock with index 0 + self.nn.exception_list[p]=e # assign exception to exception list of neural network model at process index p + loss=loss.numpy() # convert loss to numpy array + if len(self.nn.train_loss_list)==self.nn.max_length: # if train loss list of neural network model has reached the maximum length + del self.nn.train_loss_list[0] # delete the first element of train loss list of neural network model + self.nn.train_loss_list.append(loss) # append loss to train loss list of neural network model try: - if hasattr(self.nn,'counter'): # check if neural network model has a counter attribute (training counter) - count=self.nn.counter[p] # get the training counter for current process - count+=1 # increment the training counter by 1 - self.nn.counter[p]=count # assign the training counter back to neural network model + if hasattr(self.nn,'counter'): # if neural network model has an attribute called counter + count=self.nn.counter[p] # get count value from counter list of neural network model at process index p + count+=1 # increment count value by 1 + self.nn.counter[p]=count # assign count value back to counter list of neural network model at process index p except IndexError: - self.nn.counter.append(0) # append a zero to indicate that current process has not started training yet - count=self.nn.counter[p] # get the training counter for current process - count+=1 # increment the training counter by 1 - self.nn.counter[p]=count # assign the training counter back to neural network model - return + self.nn.counter.append(0) # if index error occurs, append zero to counter list of neural network model + count=self.nn.counter[p] # get count value from counter list of neural network model at process index p + count+=1 # increment count value by 1 + self.nn.counter[p]=count # assign count value back to counter list of neural network model at process index p + return # return without any value From b438c076497314166a725bad5c5396630569bdec Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Wed, 2 Aug 2023 23:02:52 +0800 Subject: [PATCH 147/337] Update kernel.py --- Note 7.0 documentation/RL/kernel/parallel/kernel.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Note 7.0 documentation/RL/kernel/parallel/kernel.py b/Note 7.0 documentation/RL/kernel/parallel/kernel.py index ccf22769..14d92f97 100644 --- a/Note 7.0 documentation/RL/kernel/parallel/kernel.py +++ b/Note 7.0 documentation/RL/kernel/parallel/kernel.py @@ -504,7 +504,7 @@ def train(self,p,lock,pool_lock,g_lock=None): while True: # loop indefinitely until break condition is met if self.stop_flag.value==True: # if stop flag is True break # break out of the loop - if self.episode_counter.value>=self.episode: # if episode counter value is greater than or equal to episode + if self.episode!=None and self.episode_counter.value>=self.episode: # if episode counter value is greater than or equal to episode break # break out of the loop s=self.nn.env(p=p,initial=True) # get initial state from calling env attribute of neural network model with process index and initial flag as arguments if type(self.nn.param[0])!=list: # if neural network model parameter is not a list @@ -513,6 +513,8 @@ def train(self,p,lock,pool_lock,g_lock=None): s=np.array(s,self.nn.param[0][0].dtype.name) # convert state data to the same data type as the first element of the neural network model parameter list if self.episode_step==None: # if episode step value is not given while True: # loop indefinitely until break condition is met + if self.episode!=None and self.episode_counter.value>=self.episode: # if episode counter value is greater than or equal to episode + break # break out of the loop next_s,r,done,index=self.env(s,epsilon,p,lock,pool_lock) # get next state, reward, done flag and index from calling env method with state, epsilon, process index, lock and pool lock as arguments self.reward[p]+=r # add reward to reward array for process index p s=next_s # set state to next state @@ -535,7 +537,7 @@ def train(self,p,lock,pool_lock,g_lock=None): break # break out of the loop else: # otherwise, assume that episode step value is given for l in range(self.episode_step): # loop through the episode step indices - if self.episode_counter.value>=self.episode: # if episode counter value is greater than or equal to episode + if self.episode!=None and self.episode_counter.value>=self.episode: # if episode counter value is greater than or equal to episode break # break out of the loop next_s,r,done,index=self.env(s,epsilon,p,lock,pool_lock) # get next state, reward, done flag and index from calling env method with state, epsilon, process index, lock and pool lock as arguments self.reward[p]+=r # add reward to reward array for process index p From 9438346b9661de973f6c5bc13de77244b82d93b4 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 3 Aug 2023 22:15:10 +0800 Subject: [PATCH 148/337] Update kernel_pytorch.py --- .../RL/kernel/parallel/kernel_pytorch.py | 698 +++++++++--------- 1 file changed, 342 insertions(+), 356 deletions(-) diff --git a/Note 7.0 documentation/RL/kernel/parallel/kernel_pytorch.py b/Note 7.0 documentation/RL/kernel/parallel/kernel_pytorch.py index 25cc2d67..b8fbd3ba 100644 --- a/Note 7.0 documentation/RL/kernel/parallel/kernel_pytorch.py +++ b/Note 7.0 documentation/RL/kernel/parallel/kernel_pytorch.py @@ -7,440 +7,426 @@ class kernel: def __init__(self,nn=None,process=None,device='GPU'): - self.nn=nn # the neural network model for the agent + self.nn=nn # the neural network to be trained if process!=None: self.reward=np.zeros(process,dtype=np.float32) # the reward array for each process self.sc=np.zeros(process,dtype=np.int32) # the step counter array for each process - self.device=device # the device to run the model on, either GPU or CPU - self.epsilon=None # the epsilon value for the epsilon-greedy policy + self.device=device # the device to use, GPU or CPU + self.epsilon=None # the epsilon value for epsilon-greedy policy self.episode_step=None # the maximum number of steps per episode - self.pool_size=None # the maximum size of the state pool for each process + self.pool_size=None # the size of the pool to store state, action, reward and done + self.episode=None # the maximum number of episodes to run self.batch=None # the batch size for training - self.update_step=None # the frequency of updating the target network parameters - self.trial_count=None # the number of trials to calculate the average reward + self.update_step=None # the update step for target network + self.trial_count=None # the number of trials to calculate average reward self.process=process # the number of processes to run in parallel - self.priority_flag=False # the flag to indicate whether to use priority-based optimization or not - self.max_opt=None # the maximum number of optimization steps for a priority process - self.stop=False # the flag to indicate whether to stop training or not - self.opt_counter=None # the array to store the optimization counter for each process - self.s=None # a temporary variable to store a state tensor - self.filename='save.dat' # the file name to save and load the model parameters and states + self.priority_flag=False # the flag to indicate whether to use priority-based optimization + self.max_opt=None # the maximum number of optimization steps per process + self.stop=False # the flag to indicate whether to stop training + self.s=None # the state variable for online training + self.filename='save.dat' # the file name to save the model def init(self,manager): - """This method is used to initialize some shared variables using a manager object""" - - self.state_pool=manager.dict({}) - self.action_pool=manager.dict({}) - self.next_state_pool=manager.dict({}) - self.reward_pool=manager.dict({}) - self.done_pool=manager.dict({}) - self.reward=Array('f',self.reward) - self.loss=np.zeros(self.process) - self.loss=Array('f',self.loss) - self.sc=Array('i',self.sc) - self.process_counter=Value('i',0) - self.probability_list=manager.list([]) - self.running_flag_list=manager.list([]) - self.finish_list=manager.list([]) - self.running_flag=manager.list([0]) - self.reward_list=manager.list([]) - self.loss_list=manager.list([]) - self.total_episode=Value('i',0) - self.priority_p=Value('i',0) - if self.priority_flag==True: - self.opt_counter=Array('i',np.zeros(self.process,dtype=np.int32)) + self.state_pool=manager.dict({}) # a dictionary to store state pool for each process + self.action_pool=manager.dict({}) # a dictionary to store action pool for each process + self.next_state_pool=manager.dict({}) # a dictionary to store next state pool for each process + self.reward_pool=manager.dict({}) # a dictionary to store reward pool for each process + self.done_pool=manager.dict({}) # a dictionary to store done pool for each process + self.reward=Array('f',self.reward) # a shared array to store reward for each process + self.loss=np.zeros(self.process) # an array to store loss for each process + self.loss=Array('f',self.loss) # a shared array to store loss for each process + self.sc=Array('i',self.sc) # a shared array to store step counter for each process + self.process_counter=Value('i',0) # a shared value to count the number of running processes + self.probability_list=manager.list([]) # a list to store probability distribution for selecting index from pool + self.running_flag_list=manager.list([]) # a list to store running flag list for each process + self.finish_list=manager.list([]) # a list to store finished processes' indices + self.running_flag=manager.list([0]) # a list to store running flag for each process, initialized with 0 + self.reward_list=manager.list([]) # a list to store reward history + self.loss_list=manager.list([]) # a list to store loss history + self.episode_counter=Value('i',0) # a shared value to count the number of episodes completed by all processes + self.total_episode=Value('i',0) # a shared value to count the total number of episodes completed by all processes + self.priority_p=Value('i',0) # a shared value to indicate which process has priority for optimization + if self.priority_flag==True: + self.opt_counter=Array('i',np.zeros(self.process,dtype=np.int32)) # an array to count optimization steps for each process try: - self.nn.opt_counter=manager.list([self.nn.opt_counter]) + self.nn.opt_counter=manager.list([self.nn.opt_counter]) # a list to store optimization counter for each neural network except Exception: - self.opt_counter_=manager.list() + self.opt_counter_=manager.list() # an empty list if no optimization counter is available try: - self.nn.ec=manager.list([self.nn.ec]) + self.nn.ec=manager.list([self.nn.ec]) # a list to store episode counter for each neural network except Exception: - self.ec_=manager.list() + self.ec_=manager.list() # an empty list if no episode counter is available try: - self.nn.bc=manager.list([self.nn.bc]) + self.nn.bc=manager.list([self.nn.bc]) # a list to store batch counter for each neural network except Exception: - self.bc_=manager.list() - self.episode_=Value('i',self.total_episode.value) - self.stop_flag=Value('b',False) - self.save_flag=Value('b',False) - self.file_list=manager.list([]) + self.bc_=manager.list() # an empty list if no batch counter is available + self.episode_=Value('i',self.total_episode.value) # a shared value to copy total episode value + self.stop_flag=Value('b',False) # a shared value to indicate whether to stop training + self.save_flag=Value('b',False) # a shared value to indicate whether to save the model + self.file_list=manager.list([]) # a list to store file names for saving the model return def init_online(self,manager): - """This method is used to initialize some shared variables for online learning using a manager object""" - - self.nn.train_loss_list=manager.list([]) - self.nn.counter=manager.list([]) - self.nn.exception_list=manager.list([]) + self.nn.train_loss_list=manager.list([]) # a list to store training loss for online training + self.nn.counter=manager.list([]) # a list to store counter for online training + self.nn.exception_list=manager.list([]) # a list to store exceptions for online training return def action_vec(self): - """This method is used to create a vector of ones with the same length as the action space""" - - self.action_one=np.ones(self.action_count,dtype=np.int8) + self.action_one=np.ones(self.action_count,dtype=np.int8) # an array of ones with the same size as action space return def set_up(self,epsilon=None,episode_step=None,pool_size=None,batch=None,update_step=None,trial_count=None,criterion=None): - """This method is used to set up some hyperparameters for the agent""" - if epsilon!=None: - self.epsilon=np.ones(self.process)*epsilon # set the epsilon value for each process + self.epsilon=np.ones(self.process)*epsilon # an array of epsilon values for each process + self.action_vec() # create the action vector if episode_step!=None: - self.episode_step=episode_step # set the maximum number of steps per episode + self.episode_step=episode_step # set the maximum number of steps per episode if pool_size!=None: - self.pool_size=pool_size # set the maximum size of the state pool for each process + self.pool_size=pool_size # set the size of the pool if batch!=None: - self.batch=batch # set the batch size for training + self.batch=batch # set the batch size for training if update_step!=None: - self.update_step=update_step # set the frequency of updating the target network parameters + self.update_step=update_step # set the update step for target network if trial_count!=None: - self.trial_count=trial_count # set the number of trials to calculate the average reward + self.trial_count=trial_count # set the number of trials to calculate average reward if criterion!=None: - self.criterion=criterion # set the criterion to stop training when the average reward reaches it - if epsilon!=None: - self.action_vec() # create a vector of ones with the same length as the action space + self.criterion=criterion # set the criterion for stopping training based on average reward return def epsilon_greedy_policy(self,s,epsilon,p): - """This method is used to implement an epsilon-greedy policy for action selection""" - - action_prob=self.action_one*epsilon/len(self.action_one) # initialize a uniform distribution over actions - s=torch.tensor(s,dtype=torch.float).to(assign_device(p,self.device)) # convert the state to a tensor and assign it to a device - best_a=self.nn.nn(s).argmax() # get the best action according to the network output - action_prob[best_a.numpy()]+=1-epsilon # increase the probability of the best action by 1-epsilon - return action_prob # return the action probability vector + action_prob=self.action_one*epsilon/len(self.action_one) # create a uniform probability distribution for actions + s=torch.tensor(s,dtype=torch.float).to(assign_device(p,self.device)) # convert state to tensor and assign device + best_a=self.nn.nn(s).argmax() # get the best action from neural network output + action_prob[best_a.numpy()]+=1-epsilon # increase the probability of best action by (1-epsilon) + return action_prob def pool(self,s,a,next_s,r,done,pool_lock,index): - """This method is used to store the state transitions in the state pool for a given process index""" - - pool_lock[index].acquire() # acquire a lock for the process index + pool_lock[index].acquire() # acquire lock for index try: - if type(self.state_pool[index])!=np.ndarray and self.state_pool[index]==None: # if the state pool is empty for this index - self.state_pool[index]=s # store the state as an array - if type(a)==int: # if the action is an integer - a=np.array(a) # convert it to an array - self.action_pool[index]=np.expand_dims(a,axis=0) # store the action as an array with an extra dimension - else: # if the action is already an array - self.action_pool[index]=a # store the action as it is - self.next_state_pool[index]=np.expand_dims(next_s,axis=0) # store the next state as an array with an extra dimension - self.reward_pool[index]=np.expand_dims(r,axis=0) # store the reward as an array with an extra dimension - self.done_pool[index]=np.expand_dims(done,axis=0) # store the done flag as an array with an extra dimension - else: # if the state pool is not empty for this index + if type(self.state_pool[index])!=np.ndarray and self.state_pool[index]==None: # if state pool is empty + self.state_pool[index]=s # store state + if type(a)==int: + a=np.array(a) + self.action_pool[index]=np.expand_dims(a,axis=0) # store action with one dimension + else: + self.action_pool[index]=a # store action + self.next_state_pool[index]=np.expand_dims(next_s,axis=0) # store next state with one dimension + self.reward_pool[index]=np.expand_dims(r,axis=0) # store reward with one dimension + self.done_pool[index]=np.expand_dims(done,axis=0) # store done with one dimension + else: try: - self.state_pool[index]=np.concatenate((self.state_pool[index],s),0) # append the state to the existing state pool - if type(a)==int: # if the action is an integer - a=np.array(a) # convert it to an array - self.action_pool[index]=np.concatenate((self.action_pool[index],np.expand_dims(a,axis=0)),0) # append the action to the existing action pool with an extra dimension - else: # if the action is already an array - self.action_pool[index]=np.concatenate((self.action_pool[index],a),0) # append the action to the existing action pool as it is - self.next_state_pool[index]=np.concatenate((self.next_state_pool[index],np.expand_dims(next_s,axis=0)),0) # append the next state to the existing next state pool with an extra dimension - self.reward_pool[index]=np.concatenate((self.reward_pool[index],np.expand_dims(r,axis=0)),0) # append the reward to the existing reward pool with an extra dimension - self.done_pool[index]=np.concatenate((self.done_pool[index],np.expand_dims(done,axis=0)),0) # append the done flag to the existing done pool with an extra dimension - except Exception: - pass - if type(self.state_pool[index])==np.ndarray and len(self.state_pool[index])>self.pool_size: # if the state pool exceeds the maximum size for this index - self.state_pool[index]=self.state_pool[index][1:] # remove the oldest state from the state pool - self.action_pool[index]=self.action_pool[index][1:] # remove the oldest action from the action pool - self.next_state_pool[index]=self.next_state_pool[index][1:] # remove the oldest next state from the next state pool - self.reward_pool[index]=self.reward_pool[index][1:] # remove the oldest reward from the reward pool - self.done_pool[index]=self.done_pool[index][1:] # remove the oldest done flag from the done pool + self.state_pool[index]=np.concatenate((self.state_pool[index],s),0) # append state to state pool + if type(a)==int: + a=np.array(a) + self.action_pool[index]=np.concatenate((self.action_pool[index],np.expand_dims(a,axis=0)),0) # append action to action pool with one dimension + else: + self.action_pool[index]=np.concatenate((self.action_pool[index],a),0) # append action to action pool + self.next_state_pool[index]=np.concatenate((self.next_state_pool[index],np.expand_dims(next_s,axis=0)),0) # append next state to next state pool with one dimension + self.reward_pool[index]=np.concatenate((self.reward_pool[index],np.expand_dims(r,axis=0)),0) # append reward to reward pool with one dimension + self.done_pool[index]=np.concatenate((self.done_pool[index],np.expand_dims(done,axis=0)),0) # append done to done pool with one dimension + except Exception: + pass + if type(self.state_pool[index])==np.ndarray and len(self.state_pool[index])>self.pool_size: # if state pool exceeds the size limit + self.state_pool[index]=self.state_pool[index][1:] # remove the oldest state from state pool + self.action_pool[index]=self.action_pool[index][1:] # remove the oldest action from action pool + self.next_state_pool[index]=self.next_state_pool[index][1:] # remove the oldest next state from next state pool + self.reward_pool[index]=self.reward_pool[index][1:] # remove the oldest reward from reward pool + self.done_pool[index]=self.done_pool[index][1:] # remove the oldest done from done pool except Exception: - pool_lock[index].release() # release the lock for the process index + pool_lock[index].release() # release lock for index if there is an exception return - pool_lock[index].release() # release the lock for the process index + pool_lock[index].release() # release lock for index after updating the pools return def get_index(self,p,lock): - """This method is used to get a random index of a running process according to their probabilities""" - - while len(self.running_flag_list)self.process_counter.value: # if there are not enough or too many running flags for this process index - self.running_flag_list[p]=self.running_flag[1:].copy() # update the running flag list for this process index with a copy of the running flag list (without the first element) - while len(self.probability_list)self.process_counter.value: # if running flag list has less elements than process counter or more elements than process counter + self.running_flag_list[p]=self.running_flag[1:].copy() # copy the running flag list and assign it to the p-th element of the list of lists + while len(self.probability_list)=self.trial_count: # if there are enough rewards in the list - avg_reward=statistics.mean(self.reward_list[-self.trial_count:]) # calculate the average reward of the last trial count episodes - if self.criterion!=None and avg_reward>=self.criterion: # if there is a criterion for stopping and the average reward reaches it - return True # return True to indicate that training should be stopped - return False # return False to indicate that training should not be stopped + if self.trial_count!=None: # if trial count is not None + if len(self.reward_list)>=self.trial_count: # if reward list has enough elements + avg_reward=statistics.mean(self.reward_list[-self.trial_count:]) # calculate the average reward of the last trial count elements + if self.criterion!=None and avg_reward>=self.criterion: # if criterion is not None and average reward is greater than or equal to criterion + return True # return True, meaning that training should end + return False # otherwise, return False, meaning that training should continue def opt(self,state_batch,action_batch,next_state_batch,reward_batch,done_batch,p): - """This method is used to optimize the network parameters using a batch of data""" - - loss=self.nn.loss(state_batch,action_batch,next_state_batch,reward_batch,done_batch,p) # calculate the loss according to the network model and the data batch - if self.priority_flag==True and self.priority_p.value!=-1: # if priority-based optimization is enabled and there is a priority process index + loss=self.nn.loss(state_batch,action_batch,next_state_batch,reward_batch,done_batch,p) # calculate the loss from batch data and process index + if self.priority_flag==True and self.priority_p.value!=-1: # if priority flag is True and priority process index is not -1 while True: - if self.stop_flag.value==True: # if training should be stopped (shared) - return None # return None to indicate that optimization is aborted - if p==self.priority_p.value: # if this process index is equal to the priority process index - break # break out of the loop - else: # if this process index is not equal to the priority process index - continue # try again - if self.stop_func_(): # if training should be stopped (local) - return None # return None to indicate that optimization is aborted - loss=loss.clone() # clone the loss tensor to avoid modifying it in place - self.nn.backward(loss,p) # perform backpropagation on the loss tensor according to the network model and this process index - self.nn.opt(p) # perform optimization on the network parameters according to this process index - return loss # return the loss tensor + if self.stop_flag.value==True: # if stop flag is True + return None # return None, meaning that optimization should stop + if p==self.priority_p.value: # if process index is equal to priority process index + break # break the loop and continue optimization + else: + continue # otherwise, continue the loop and wait for priority + if self.stop_func_(): # if stop function returns True + return None # return None, meaning that optimization should stop + loss=loss.clone() # clone the loss tensor + self.nn.backward(loss,p) # perform backward propagation with loss and process index + self.nn.opt(p) # perform optimization with process index + return loss def _train(self,p,j,batches,length): - """This method is used to train on a batch of data from the state pool for a given process index""" - - if j==batches-1: # if this is the last batch of data - index1=batches*self.batch # get the starting index of this batch from the end of the state pool - index2=self.batch-(length-batches*self.batch) # get the ending index of this batch from the beginning of the state pool - state_batch=np.concatenate((self.state_pool[p][index1:length],self.state_pool[p][:index2]),0) # concatenate two slices of states from both ends of the state pool as a batch - action_batch=np.concatenate((self.action_pool[p][index1:length],self.action_pool[p][:index2]),0) # concatenate two slices of actions from both ends of the action pool as a batch - next_state_batch=np.concatenate((self.next_state_pool[p][index1:length],self.next_state_pool[p][:index2]),0) # concatenate two slices of next states from both ends of the next state pool as a batch - reward_batch=np.concatenate((self.reward_pool[p][index1:length],self.reward_pool[p][:index2]),0) # concatenate two slices of rewards from both ends of the reward pool as a batch - done_batch=np.concatenate((self.done_pool[p][index1:length],self.done_pool[p][:index2]),0) # concatenate two slices of done flags from both ends of the done pool as a batch - loss=self.opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch,p) # optimize the network parameters using this batch of data - self.loss[p]+=loss # accumulate the loss for this process index - if hasattr(self.nn,'bc'): # if there is a batch counter for the network model - bc=self.nn.bc[0] # get the current value of the batch counter - bc.assign_add(1) # increment the batch counter by 1 - self.nn.bc[0]=bc # update the batch counter for the network model - else: # if this is not the last batch of data - index1=j*self.batch # get the starting index of this batch from the state pool - index2=(j+1)*self.batch # get the ending index of this batch from the state pool - state_batch=self.state_pool[p][index1:index2] # get a slice of states from the state pool as a batch - action_batch=self.action_pool[p][index1:index2] # get a slice of actions from the action pool as a batch - next_state_batch=self.next_state_pool[p][index1:index2] # get a slice of next states from the next state pool as a batch - reward_batch=self.reward_pool[p][index1:index2] # get a slice of rewards from the reward pool as a batch - done_batch=self.done_pool[p][index1:index2] # get a slice of done flags from the done pool as a batch - loss=self.opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch,p) # optimize the network parameters using this batch of data - self.loss[p]+=loss # accumulate the loss for this process index - if hasattr(self.nn,'bc'): # if there is a batch counter for the network model - bc=self.nn.bc[0] # get the current value of the batch counter - bc.assign_add(1) # increment the batch counter by 1 - self.nn.bc[0]=bc # update the batch counter for the network model + if j==batches-1: # if it is the last batch + index1=batches*self.batch # get the start index of the batch + index2=self.batch-(length-batches*self.batch) # get the end index of the batch + state_batch=np.concatenate((self.state_pool[p][index1:length],self.state_pool[p][:index2]),0) # get the state batch by concatenating the last part and the first part of state pool + action_batch=np.concatenate((self.action_pool[p][index1:length],self.action_pool[p][:index2]),0) # get the action batch by concatenating the last part and the first part of action pool + next_state_batch=np.concatenate((self.next_state_pool[p][index1:length],self.next_state_pool[p][:index2]),0) # get the next state batch by concatenating the last part and the first part of next state pool + reward_batch=np.concatenate((self.reward_pool[p][index1:length],self.reward_pool[p][:index2]),0) # get the reward batch by concatenating the last part and the first part of reward pool + done_batch=np.concatenate((self.done_pool[p][index1:length],self.done_pool[p][:index2]),0) # get the done batch by concatenating the last part and the first part of done pool + loss=self.opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch,p) # get the loss from opt method with batch data and process index + self.loss[p]+=loss # add loss to loss array for process index + if hasattr(self.nn,'bc'): + bc=self.nn.bc[0] + bc.assign_add(1) + self.nn.bc[0]=bc + else: + index1=j*self.batch + index2=(j+1)*self.batch + state_batch=self.state_pool[p][index1:index2] + action_batch=self.action_pool[p][index1:index2] + next_state_batch=self.next_state_pool[p][index1:index2] + reward_batch=self.reward_pool[p][index1:index2] + done_batch=self.done_pool[p][index1:index2] + loss=self.opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch,p) + self.loss[p]+=loss + if hasattr(self.nn,'bc'): + bc=self.nn.bc[0] + bc.assign_add(1) + self.nn.bc[0]=bc return - def train_(self,p,lock): - """This method is used to train on all batches of data from the state pool for a given process index""" - - if len(self.done_pool[p])=self.max_opt: # if there is a maximum number of optimization steps for a priority process and it is reached by the priority process index - self.priority_p.value=int(self.priority_p.value) # convert the priority process index to an integer type - elif self.max_opt==None: # if there is no maximum number of optimization steps for a priority process - self.priority_p.value=int(self.priority_p.value) # convert the priority process index to an integer type - else: # if there is a maximum number of optimization steps for a priority process and it is not reached by any process index - self.priority_p.value=-1 # set the priority process index to -1 to indicate no priority - if self.priority_flag==True: # if priority-based optimization is enabled - self.opt_counter[p]=0 # reset the optimization counter for this process index - if hasattr(self.nn,'attenuate'): # if there is an attenuate method for the network model - self.nn.attenuate(p) # attenuate some parameters according to the network model and this process index - self._train(p,j,batches,length) # train on a batch of data from the state pool for this process index - if self.priority_flag==True: # if priority-based optimization is enabled - opt_counter=np.frombuffer(self.opt_counter.get_obj(),dtype='i') # get the optimization counter array as a numpy array - opt_counter+=1 # increment the optimization counter array by 1 - if hasattr(self.nn,'attenuate'): # if there is an attenuate method for the network model - opt_counter=self.nn.opt_counter[0] # get the current value of the optimization counter for the network model - opt_counter+=1 # increment the optimization counter for the network model by 1 - self.nn.opt_counter[0]=opt_counter # update the optimization counter for the network model - if self.update_step!=None: # if there is a frequency of updating the target network parameters - if self.sc[p]%self.update_step==0: # if this process index reaches the update frequency - self.nn.update_param() # update the target network parameters according to the network model - else: # if there is no frequency of updating the target network parameters - self.nn.update_param() # update the target network parameters according to the network model - self.loss[p]=self.loss[p]/batches # calculate the average loss for this process index - self.sc[p]+=1 # increment the step counter for this process index - if hasattr(self.nn,'ec'): # if there is an episode counter for the network model - ec=self.nn.ec[0] # get the current value of the episode counter - ec.assign_add(1) # increment the episode counter by 1 - self.nn.ec[0]=ec # update the episode counter for the network model + def train_(self,p): + if len(self.done_pool[p])=self.max_opt: # if max opt is not None and opt counter for priority process index is greater than or equal to max opt + self.priority_p.value=int(self.priority_p.value) # convert priority process index to integer + elif self.max_opt==None: # if max opt is None + self.priority_p.value=int(self.priority_p.value) # convert priority process index to integer + else: + self.priority_p.value=-1 # set priority process index to -1, meaning no priority + if self.priority_flag==True: + self.opt_counter[p]=0 # reset opt counter for process index to zero + if hasattr(self.nn,'attenuate'): + opt_counter=self.nn.opt_counter[0] + opt_counter[p]=0 + self.nn.opt_counter[0]=opt_counter + self._train(p,j,batches,length) # call _train method with process index, batch index, number of batches and length of done pool + if self.priority_flag==True: + opt_counter=np.frombuffer(self.opt_counter.get_obj(),dtype='i') # get the opt counter array from shared memory + opt_counter+=1 # increment the opt counter array by one + if hasattr(self.nn,'attenuate'): + opt_counter=self.nn.opt_counter[0] + opt_counter+=1 + self.nn.opt_counter[0]=opt_counter + if self.update_step!=None: # if update step is not None + if self.sc[p]%self.update_step==0: # if step counter for process index is divisible by update step + self.nn.update_param() # update the target network parameters + else: + self.nn.update_param() # update the target network parameters + self.loss[p]=self.loss[p]/batches # calculate the average loss for process index by dividing by number of batches + self.sc[p]+=1 # increment the step counter for process index by one + if hasattr(self.nn,'ec'): + ec=self.nn.ec[0] + ec.assign_add(1) + self.nn.ec[0]=ec return - def train(self,p,episode_count,lock,pool_lock): - """This method is used to start a process to train on multiple episodes""" - - lock[1].acquire() # acquire a lock for the shared variables - self.state_pool[p]=None # initialize the state pool for this process index as None - self.action_pool[p]=None # initialize the action pool for this process index as None - self.next_state_pool[p]=None # initialize the next state pool for this process index as None - self.reward_pool[p]=None # initialize the reward pool for this process index as None - self.done_pool[p]=None # initialize the done pool for this process index as None - self.running_flag.append(1) # append a 1 to the running flag list to indicate that this process is running - self.process_counter.value+=1 # increment the counter of running processes by 1 - self.finish_list.append(None) # append a None to the finish list to indicate that this process is not finished yet - lock[1].release() # release the lock for the shared variables + def train(self,p,lock,pool_lock): + lock[1].acquire() # acquire lock for running flag list + self.state_pool[p]=None # initialize state pool for process index to None + self.action_pool[p]=None # initialize action pool for process index to None + self.next_state_pool[p]=None # initialize next state pool for process index to None + self.reward_pool[p]=None # initialize reward pool for process index to None + self.done_pool[p]=None # initialize done pool for process index to None + self.running_flag.append(1) # append one to running flag list, meaning that process is running + self.process_counter.value+=1 # increment the process counter by one + self.finish_list.append(None) # append None to finish list, meaning that process is not finished + lock[1].release() # release lock for running flag list try: - epsilon=self.epsilon[p] # get the epsilon value for this process index + epsilon=self.epsilon[p] # get the epsilon value for process index except Exception: - epsilon=None # set the epsilon value to None if there is no epsilon value - for k in range(episode_count): # for each episode - s=self.nn.env(p=p,initial=True) # get an initial state from the environment according to this process index - s=np.array(s) # convert the state to an array - if self.episode_step==None: # if there is no maximum number of steps per episode - while True: # loop until done - next_s,r,done,index=self.env(s,epsilon,p,lock,pool_lock) # interact with the environment and store the state transitions in the state pool - self.reward[p]+=r # accumulate the reward for this process index - s=next_s # update the state with next state - if type(self.done_pool[p])==np.ndarray: # if there are enough data in the state pool for this process index - self.train_(p,lock) # train on all batches of data from the state pool for this process index - if done: # if done flag is True - if len(lock)==4: # if there is a fourth lock for saving and loading - lock[3].acquire() # acquire a lock for saving and loading - self.total_episode.value+=1 # increment the total number of episodes by 1 (shared) - self.loss_list.append(self.loss[p]) # append the loss for this process index to the loss list (shared) - if len(lock)==4: # if there is a fourth lock for saving and loading - lock[3].acquire() # acquire a lock for saving and loading - self.total_episode.value+=1 # increment the total number of episodes by 1 (shared) - self.loss_list.append(self.loss[p]) # append the loss for this process index to the loss list (shared) - if len(lock)==4: # if there is a fourth lock for saving and loading - lock[3].release() # release the lock for saving and loading - break # break out of the loop - else: # if there is a maximum number of steps per episode - for l in range(self.episode_step): # for each step - next_s,r,done,index=self.env(s,epsilon,p,lock,pool_lock) # interact with the environment and store the state transitions in the state pool - self.reward[p]+=r # accumulate the reward for this process index - s=next_s # update the state with next state - if type(self.done_pool[p])==np.ndarray: # if there are enough data in the state pool for this process index - self.train_(p,lock) # train on all batches of data from the state pool for this process index - if done: # if done flag is True - if len(lock)==4: # if there is a fourth lock for saving and loading - lock[3].acquire() # acquire a lock for saving and loading - self.total_episode.value+=1 # increment the total number of episodes by 1 (shared) - self.loss_list.append(self.loss[p]) # append the loss for this process index to the loss list (shared) - if len(lock)==4: # if there is a fourth lock for saving and loading - lock[3].release() # release the lock for saving and loading - break # break out of the loop - if l==self.episode_step-1: # if this is the last step of the episode - if len(lock)==4: # if there is a fourth lock for saving and loading - lock[3].acquire() # acquire a lock for saving and loading - self.total_episode.value+=1 # increment the total number of episodes by 1 (shared) - self.loss_list.append(self.loss[p]) # append the loss for this process index to the loss list (shared) - if len(lock)==4: # if there is a fourth lock for saving and loading - lock[3].release() # release the lock for saving and loading - if len(lock)==3 or len(lock)==4: # if there is a third or fourth lock for saving and loading - lock[2].acquire() # acquire a lock for saving and loading - self.save_() # save the model parameters and states according to the network model - self.reward_list.append(self.reward[p]) # append the reward for this process index to the reward list (shared) - self.reward[p]=0 # reset the reward for this process index - if len(lock)==3 or len(lock)==4: # if there is a third or fourth lock for saving and loading - lock[2].release() # release the lock for saving and loading - self.running_flag[p+1]=0 # set the running flag to 0 to indicate that this process is not running anymore - if p not in self.finish_list: # if this process index is not in the finish list yet - self.finish_list[p]=p # add this process index to the finish list - lock[1].acquire() # acquire a lock for the shared variables - self.process_counter.value-=1 # decrement the counter of running processes by 1 - lock[1].release() # release the lock for the shared variables - del self.state_pool[p] # delete the state pool for this process index - del self.action_pool[p] # delete the action pool for this process index - del self.next_state_pool[p] # delete the next state pool for this process index - del self.reward_pool[p] # delete the reward pool for this process index - del self.done_pool[p] # delete the done pool for this process index + epsilon=None # set epsilon value to None if there is an exception + while True: + if self.stop_flag.value==True: # if stop flag is True + break # break the loop and stop training + if self.episode!=None and self.episode_counter.value>=self.episode: # if episode limit is not None and episode counter is greater than or equal to episode limit + break # break the loop and stop training + s=self.nn.env(p=p,initial=True) # get the initial state from environment with process index + s=np.array(s) + if self.episode_step==None: # if episode step limit is None + while True: + if self.episode!=None and self.episode_counter.value>=self.episode: # if episode limit is not None and episode counter is greater than or equal to episode limit + break # break the loop and stop training + next_s,r,done,index=self.env(s,epsilon,p,lock,pool_lock) # get the next state, reward, done and index from env method with state, epsilon, process index, lock and pool lock + self.reward[p]+=r # add reward to reward array for process index + s=next_s # assign next state to state + if type(self.done_pool[p])==np.ndarray: # if done pool is not empty + self.train_(p) # call train_ method with process index + if self.stop_flag.value==True: # if stop flag is True + break # break the loop and stop training + if done: # if done is True, meaning that episode is finished + if len(lock)==4: # if there are four locks + lock[3].acquire() # acquire the fourth lock for episode counter + self.episode_counter.value+=1 # increment the episode counter by one + self.total_episode.value+=1 # increment the total episode by one + self.loss_list.append(self.loss[p]) # append the loss for process index to loss list + if len(lock)==4: # if there are four locks + lock[3].release() # release the fourth lock for episode counter + break # break the loop and start a new episode + else: # if episode step limit is not None + for l in range(self.episode_step): # loop over episode step limit + if self.episode!=None and self.episode_counter.value>=self.episode: # if episode limit is not None and episode counter is greater than or equal to episode limit + break # break the loop and stop training + next_s,r,done,index=self.env(s,epsilon,p,lock,pool_lock) # get the next state, reward, done and index from env method with state, epsilon, process index, lock and pool lock + self.reward[p]+=r # add reward to reward array for process index + s=next_s # assign next state to state + if type(self.done_pool[p])==np.ndarray: # if done pool is not empty + self.train_(p) # call train_ method with process index + if self.stop_flag.value==True: # if stop flag is True + break # break the loop and stop training + if done: # if done is True, meaning that episode is finished + if len(lock)==4: # if there are four locks + lock[3].acquire() # acquire the fourth lock for episode counter + self.episode_counter.value+=1 # increment the episode counter by one + self.total_episode.value+=1 # increment the total episode by one + self.loss_list.append(self.loss[p]) # append the loss for process index to loss list + if len(lock)==4: # if there are four locks + lock[3].release() # release the fourth lock for episode counter + break # break the loop and start a new episode + if l==self.episode_step-1: # if it is the last step of episode + if len(lock)==4: # if there are four locks + lock[3].acquire() # acquire the fourth lock for episode counter + self.episode_counter.value+=1 + self.total_episode.value+=1 # increment the total episode by one + self.loss_list.append(self.loss[p]) # append the loss for process index to loss list + if len(lock)==4: # if there are four locks + lock[3].release() # release the fourth lock for episode counter + if len(lock)==3 or len(lock)==4: # if there are three or four locks + lock[2].acquire() # acquire the third lock for saving model + self.save_() # call save_ method to save the model + self.reward_list.append(self.reward[p]) # append the reward for process index to reward list + self.reward[p]=0 # reset the reward for process index to zero + if len(lock)==3 or len(lock)==4: # if there are three or four locks + lock[2].release() # release the third lock for saving model + self.running_flag[p+1]=0 # set the running flag for process index to zero, meaning that process is not running + if p not in self.finish_list: # if process index is not in finish list + self.finish_list[p]=p # add process index to finish list, meaning that process is finished + lock[1].acquire() # acquire lock for running flag list + self.process_counter.value-=1 # decrement the process counter by one + lock[1].release() # release lock for running flag list + del self.state_pool[p] # delete state pool for process index + del self.action_pool[p] # delete action pool for process index + del self.next_state_pool[p] # delete next state pool for process index + del self.reward_pool[p] # delete reward pool for process index + del self.done_pool[p] # delete done pool for process index return def train_online(self,p,lock=None,g_lock=None): - """This method is used to implement online learning using a single process""" - - if hasattr(self.nn,'counter'): # if there is a counter variable for the network model - self.nn.counter.append(0) # append a 0 to the counter list - while True: # loop until stopped - if hasattr(self.nn,'save'): # if there is a save method for the network model - self.nn.save(self.save,p) # save the model parameters and states according to the network model and this process index - if hasattr(self.nn,'stop_flag'): # if there is a stop flag for the network model - if self.nn.stop_flag==True: # if the stop flag is True - return # return to indicate that online learning is stopped - if hasattr(self.nn,'stop_func'): # if there is a stop function for the network model - if self.nn.stop_func(p): # if the stop function returns True according to this process index - return # return to indicate that online learning is stopped - if hasattr(self.nn,'suspend_func'): # if there is a suspend function for the network model - self.nn.suspend_func(p) # suspend some operations according to the network model and this process index + if hasattr(self.nn,'counter'): + self.nn.counter.append(0) + while True: + if hasattr(self.nn,'save'): + self.nn.save(self.save,p) + if hasattr(self.nn,'stop_flag'): + if self.nn.stop_flag==True: + return + if hasattr(self.nn,'stop_func'): + if self.nn.stop_func(p): + return + if hasattr(self.nn,'suspend_func'): + self.nn.suspend_func(p) try: - data=self.nn.online(p) # get some data from the online source according to the network model and this process index - except Exception as e: # if there is an exception raised - self.nn.exception_list[p]=e # append the exception to the exception list for this process index - if data=='stop': # if the data indicates to stop online learning - return # return to indicate that online learning is stopped - elif data=='suspend': # if the data indicates to suspend some operations - self.nn.suspend_func(p) # suspend some operations according to the network model and this process index + data=self.nn.online(p) # get the online data from neural network with process index + except Exception as e: + self.nn.exception_list[p]=e # store the exception in exception list for process index + if data=='stop': # if data is 'stop', meaning that online training should stop + return + elif data=='suspend': # if data is 'suspend', meaning that online training should suspend + self.nn.suspend_func(p) # call suspend function with process index try: - loss,param=self.opt(data[0],data[1],data[2],data[3],data[4],p,lock,g_lock) # optimize the network parameters using the data and some locks - except Exception as e: # if there is an exception raised - self.nn.exception_list[p]=e # append the exception to the exception list for this process index - loss=loss.numpy() # convert the loss tensor to a numpy array - if len(self.nn.train_loss_list)==self.nn.max_length: # if the train loss list reaches its maximum length - del self.nn.train_loss_list[0] # delete the oldest element from the train loss list - self.nn.train_loss_list.append(loss) # append the loss to the train loss list + loss,param=self.opt(data[0],data[1],data[2],data[3],data[4],p,lock,g_lock) # get the loss and parameter from opt method with data, process index, lock and global lock + except Exception as e: + self.nn.exception_list[p]=e # store the exception in exception list for process index + loss=loss.numpy() + if len(self.nn.train_loss_list)==self.nn.max_length: # if train loss list has reached the maximum length + del self.nn.train_loss_list[0] # delete the oldest element from train loss list + self.nn.train_loss_list.append(loss) # append the loss to train loss list try: - if hasattr(self.nn,'counter'): # if there is a counter variable for the network model - count=self.nn.counter[p] # get the current value of the counter for this process index - count+=1 # increment the counter by 1 - self.nn.counter[p]=count # update the counter for this process index - except IndexError: # if there is no counter for this process index yet - self.nn.counter.append(0) # append a 0 to the counter list - count=self.nn.counter[p] # get the current value of the counter for this process index - count+=1 # increment the counter by 1 - self.nn.counter[p]=count # update the counter for this process index + if hasattr(self.nn,'counter'): + count=self.nn.counter[p] + count+=1 + self.nn.counter[p]=count + except IndexError: + self.nn.counter.append(0) + count=self.nn.counter[p] + count+=1 + self.nn.counter[p]=count return From f114f8ac92dbc4ea0070cb8a35152121f0c6bbb9 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Fri, 4 Aug 2023 14:04:23 +0800 Subject: [PATCH 149/337] Update README.md --- README.md | 60 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/README.md b/README.md index 79bd24fe..d2a199fc 100644 --- a/README.md +++ b/README.md @@ -312,6 +312,66 @@ for p in range(3): #loop over the processes Process(target=kernel.train,args=(p,lock)).start() #start each process with the train function and pass the process id and locks as arguments ``` +### Process priority: +```python +import Note.DL.parallel.kernel as k #import kernel module +import tensorflow as tf #import tensorflow library +import nn as n #import neural network module +from multiprocessing import Process,Manager #import multiprocessing tools +mnist=tf.keras.datasets.mnist #load mnist dataset +(x_train,y_train),(x_test,y_test)=mnist.load_data() #split data into train and test sets +x_train,x_test =x_train/255.0,x_test/255.0 #normalize data +nn=n.nn() #create neural network object +nn.build() #build the network structure +kernel=k.kernel(nn) #create kernel object with the network +kernel.process=3 #set the number of processes to train +kernel.data_segment_flag=True #set the flag to segment data for each process +kernel.epoch=5 #set the number of epochs to train +kernel.batch=32 #set the batch size +kernel.priority_flag=True #set the flag to use priority scheduling for processes +kernel.PO=3 #use PO3 algorithm for parallel optimization +kernel.data(x_train,y_train) #input train data to the kernel +manager=Manager() #create manager object to share data among processes +kernel.init(manager) #initialize shared data with the manager +for p in range(3): #loop over the processes + Process(target=kernel.train,args=(p,)).start() #start each process with the train function and pass the process id and locks as arguments +kernel.update_nn_param() #update the network parameters after training +kernel.test(x_train,y_train,32) #test the network performance on the train set with batch size 32 +``` + +### Gradient attenuation: +**Calculate the attenuation coefficient based on the optimization counter using the attenuation function.** + +**You can get neural network example from the link below, and then you can import neural network and train with kernel, example code are below.** + +https://github.com/NoteDancing/Note-documentation/blob/Note-7.0/Note%207.0%20documentation/DL/neural%20network/tensorflow/parallel/nn_attenuate.py +```python +import Note.DL.parallel.kernel as k #import kernel module +import tensorflow as tf #import tensorflow library +import nn_attenuate as n #import neural network module +from multiprocessing import Process,Manager #import multiprocessing tools +mnist=tf.keras.datasets.mnist #load mnist dataset +(x_train,y_train),(x_test,y_test)=mnist.load_data() #split data into train and test sets +x_train,x_test =x_train/255.0,x_test/255.0 #normalize data +nn=n.nn() #create neural network object +nn.build() #build the network structure +kernel=k.kernel(nn) #create kernel object with the network +kernel.process=3 #set the number of processes to train +kernel.data_segment_flag=True #set the flag to segment data for each process +kernel.epoch=5 #set the number of epochs to train +kernel.batch=32 #set the batch size +kernel.priority_flag=True #set the flag to use priority scheduling for processes +kernel.PO=3 #use PO3 algorithm for parallel optimization +kernel.data(x_train,y_train) #input train data to the kernel +manager=Manager() #create manager object to share data among processes +kernel.init(manager) #initialize shared data with the manager +for p in range(3): #loop over the processes + Process(target=kernel.train,args=(p,)).start() #start each process with the train function and pass the process id and locks as arguments +kernel.update_nn_param() #update the network parameters after training +kernel.test(x_train,y_train,32) #test the network performance on the train set with batch size 32 +``` + + ## RL： **Pool Network:** From d9f9ddfcb2122ca35e918cb699319045a67b8909 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Fri, 4 Aug 2023 14:21:35 +0800 Subject: [PATCH 150/337] Update README.md --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index d2a199fc..18590649 100644 --- a/README.md +++ b/README.md @@ -525,3 +525,9 @@ import DQN as d #import deep Q-network module dqn=d.DQN(4,128,2) #create neural network object with 4 inputs, 128 hidden units and 2 outputs check(dqn,tf,2) #check the network with tensorflow platform and 2 actions ``` + + +# GPT: +**Layers in the GPT directory created by GPT** + +https://github.com/NoteDancing/Note/tree/Note-7.0/Note/nn/layer/GPT From b1e118fc7956d8f747c35cdae31066a3325bd863 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Fri, 4 Aug 2023 22:31:47 +0800 Subject: [PATCH 151/337] Update README.md --- README.md | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 18590649..fcc5aa7e 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ kernel=k.kernel(dqn) #create kernel object with the network kernel.stop=True #set the flag to stop training when a condition is met kernel.action_count=2 #set the number of actions to 2 kernel.set_up(epsilon=0.01,pool_size=10000,batch=64,update_step=10,trial_count=10,criterion=200) #set up the hyperparameters for training and the condition to stop training when the average reward of 10 trials is greater than 200 -kernel.train(500) #train the network for 500 episodes +kernel.train(100) #train the network for 500 episodes ``` ## Training with test data @@ -386,7 +386,7 @@ import DQN as d #import deep Q-network module from multiprocessing import Process,Lock,Manager #import multiprocessing tools dqn=d.DQN(4,128,2) #create neural network object with 4 inputs, 128 hidden units and 2 outputs kernel=k.kernel(dqn,5) #create kernel object with the network and 5 processes to train -kernel.episode=300 #set the number of episodes to 300 +kernel.episode=100 #set the number of episodes to 100 manager=Manager() #create manager object to share data among processes kernel.init(manager) #initialize shared data with the manager kernel.action_count=2 #set the number of actions to 2 @@ -405,7 +405,7 @@ import DQN as d #import deep Q-network module from multiprocessing import Process,Lock,Manager #import multiprocessing tools dqn=d.DQN(4,128,2) #create neural network object with 4 inputs, 128 hidden units and 2 outputs kernel=k.kernel(dqn,5) #create kernel object with the network and 5 processes to train -kernel.episode=300 #set the number of episodes to 300 +kernel.episode=100 #set the number of episodes to 100 manager=Manager() #create manager object to share data among processes kernel.init(manager) #initialize shared data with the manager kernel.action_count=2 #set the number of actions to 2 @@ -425,7 +425,7 @@ import DQN as d #import deep Q-network module from multiprocessing import Process,Lock,Manager #import multiprocessing tools dqn=d.DQN(4,128,2) #create neural network object with 4 inputs, 128 hidden units and 2 outputs kernel=k.kernel(dqn,5) #create kernel object with the network and 5 processes to train -kernel.episode=300 #set the number of episodes to 300 +kernel.episode=100 #set the number of episodes to 100 manager=Manager() #create manager object to share data among processes kernel.init(manager) #initialize shared data with the manager kernel.action_count=2 #set the number of actions to 2 @@ -444,7 +444,7 @@ import DQN as d #import deep Q-network module from multiprocessing import Process,Lock,Manager #import multiprocessing tools dqn=d.DQN(4,128,2) #create neural network object with 4 inputs, 128 hidden units and 2 outputs kernel=k.kernel(dqn,5) #create kernel object with the network and 5 processes to train -kernel.episode=300 #set the number of episodes to 300 +kernel.episode=100 #set the number of episodes to 100 manager=Manager() #create manager object to share data among processes kernel.init(manager) #initialize shared data with the manager kernel.action_count=2 #set the number of actions to 2 @@ -457,6 +457,26 @@ for p in range(5): #loop over the processes Process(target=kernel.train,args=(p,lock,pool_lock)).start() #start each process with the train function and pass the process id, the number of episodes, the locks and the pool locks as arguments ``` +### Process priority: +```python +import Note.RL.parallel.kernel as k #import kernel module +import DQN as d #import deep Q-network module +from multiprocessing import Process,Lock,Manager #import multiprocessing tools +dqn=d.DQN(4,128,2) #create neural network object with 4 inputs, 128 hidden units and 2 outputs +kernel=k.kernel(dqn,5) #create kernel object with the network and 5 processes to train +kernel.episode=100 #set the number of episodes to 100 +manager=Manager() #create manager object to share data among processes +kernel.priority_flag=True #set the flag to use priority scheduling for processes +kernel.init(manager) #initialize shared data with the manager +kernel.action_count=2 #set the number of actions to 2 +kernel.set_up(epsilon=0.01,pool_size=10000,batch=64,update_step=10) #set up the hyperparameters for training +kernel.PO=3 #use PO3 algorithm for parallel optimization +pool_lock=[Lock(),Lock(),Lock(),Lock(),Lock()] #create a list of locks for each process's replay pool +lock=[Lock(),Lock()] #create two locks for synchronization +for p in range(5): #loop over the processes + Process(target=kernel.train,args=(p,lock,pool_lock)).start() #start each process with the train function and pass the process id, the number of episodes, the locks and the pool locks as arguments +``` + # Parallel test: **You can get neural network example from the link below, and then you can import neural network and train with kernel, example code are below.** From 512861df0e1cde0e09e6f467090cfb6acbe266d5 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Sun, 6 Aug 2023 23:01:12 +0800 Subject: [PATCH 152/337] Update README.md --- README.md | 94 ++++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 87 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index fcc5aa7e..a2f375b5 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Non-parallel training: - -## Save and restore: +## DL: +### Save and restore: ```python import Note.DL.kernel as k #import kernel module import tensorflow as tf #import tensorflow library @@ -28,8 +28,24 @@ kernel.restore('save.dat') #restore the network from a file kernel.train(32,1) #train the network again with batch size 32 and epoch 1 ``` -## Stop training and saving when condition is met: -### DL: +### Stop training and saving when condition is met: +```python +import Note.DL.kernel as k #import kernel module +import tensorflow as tf #import tensorflow library +import nn as n #import neural network module +mnist=tf.keras.datasets.mnist #load mnist dataset +(x_train,y_train),(x_test,y_test)=mnist.load_data() #split data into train and test sets +x_train,x_test =x_train/255.0,x_test/255.0 #normalize data +nn=n.nn() #create neural network object +kernel=k.kernel(nn) #create kernel object with the network +kernel.platform=tf #set the platform to tensorflow +kernel.stop=True #set the flag to stop training when a condition is met +kernel.end_loss=0.7 #set the condition to stop training when the loss is less than 0.7 +kernel.data(x_train,y_train) #input train data to the kernel +kernel.train(32,5) #train the network with batch size 32 and epoch 5 +``` + +### Visualization: ```python import Note.DL.kernel as k #import kernel module import tensorflow as tf #import tensorflow library @@ -44,9 +60,24 @@ kernel.stop=True #set the flag to stop training when a condition is kernel.end_loss=0.7 #set the condition to stop training when the loss is less than 0.7 kernel.data(x_train,y_train) #input train data to the kernel kernel.train(32,5) #train the network with batch size 32 and epoch 5 +kernel.visualize_train() #visualize the loss ``` -### RL: + +## RL: +### Stop training and saving when condition is met: +```python +import Note.RL.kernel as k #import kernel module +import DQN as d #import deep Q-network module +dqn=d.DQN(4,128,2) #create neural network object with 4 inputs, 128 hidden units and 2 outputs +kernel=k.kernel(dqn) #create kernel object with the network +kernel.stop=True #set the flag to stop training when a condition is met +kernel.action_count=2 #set the number of actions to 2 +kernel.set_up(epsilon=0.01,pool_size=10000,batch=64,update_step=10,trial_count=10,criterion=200) #set up the hyperparameters for training and the condition to stop training when the average reward of 10 trials is greater than 200 +kernel.train(100) #train the network for 500 episodes +``` + +### Visualization: ```python import Note.RL.kernel as k #import kernel module import DQN as d #import deep Q-network module @@ -56,6 +87,8 @@ kernel.stop=True #set the flag to stop training when a condition is kernel.action_count=2 #set the number of actions to 2 kernel.set_up(epsilon=0.01,pool_size=10000,batch=64,update_step=10,trial_count=10,criterion=200) #set up the hyperparameters for training and the condition to stop training when the average reward of 10 trials is greater than 200 kernel.train(100) #train the network for 500 episodes +kernel.visualize_reward() #visualize the reward +kernel.visualize_train() #visualize the loss ``` ## Training with test data @@ -98,9 +131,7 @@ kernel.train(32,5,32) #train the network with batch size 32, epoch 5 and # Parallel training: - ## DL: - ### PO1: ```python import Note.DL.parallel.kernel as k #import kernel module @@ -259,6 +290,34 @@ for p in range(3): #loop over the processes Process(target=kernel.train,args=(p,)).start() #start each process with the train function and pass the process id as argument ``` +### Visualization: +```python +import Note.DL.parallel.kernel as k #import kernel module +import tensorflow as tf #import tensorflow library +import nn as n #import neural network module +from multiprocessing import Process,Lock,Manager #import multiprocessing tools +mnist=tf.keras.datasets.mnist #load mnist dataset +(x_train,y_train),(x_test,y_test)=mnist.load_data() #split data into train and test sets +x_train,x_test =x_train/255.0,x_test/255.0 #normalize data +nn=n.nn() #create neural network object +nn.build() #build the network structure +kernel=k.kernel(nn) #create kernel object with the network +kernel.process=3 #set the number of processes to train +kernel.data_segment_flag=True #set the flag to segment data for each process +kernel.epoch=5 #set the number of epochs to train +kernel.batch=32 #set the batch size +kernel.PO=3 #use PO3 algorithm for parallel optimization +kernel.data(x_train,y_train) #input train data to the kernel +manager=Manager() #create manager object to share data among processes +kernel.init(manager) #initialize shared data with the manager +lock=Lock() #create a lock for synchronization +for p in range(3): #loop over the processes + Process(target=kernel.train,args=(p,lock)).start() #start each process with the train function and pass the process id and the lock as arguments +kernel.update_nn_param() #update the network parameters after training +kernel.test(x_train,y_train,32) #test the network performance on the train set with batch size 32 +kernel.visualize_train() #visualize the loss +``` + ### Parallel test: ```python import Note.DL.parallel.kernel as k #import kernel module @@ -437,6 +496,27 @@ for p in range(5): #loop over the processes Process(target=kernel.train,args=(p,lock,pool_lock)).start() #start each process with the train function and pass the process id, the number of episodes, the locks and the pool locks as arguments ``` +### Visualization: +```python +import Note.RL.parallel.kernel as k #import kernel module +import DQN as d #import deep Q-network module +from multiprocessing import Process,Lock,Manager #import multiprocessing tools +dqn=d.DQN(4,128,2) #create neural network object with 4 inputs, 128 hidden units and 2 outputs +kernel=k.kernel(dqn,5) #create kernel object with the network and 5 processes to train +kernel.episode=100 #set the number of episodes to 100 +manager=Manager() #create manager object to share data among processes +kernel.init(manager) #initialize shared data with the manager +kernel.action_count=2 #set the number of actions to 2 +kernel.set_up(epsilon=0.01,pool_size=10000,batch=64,update_step=10) #set up the hyperparameters for training +kernel.PO=3 #use PO3 algorithm for parallel optimization +pool_lock=[Lock(),Lock(),Lock(),Lock(),Lock()] #create a list of locks for each process's replay pool +lock=[Lock(),Lock(),Lock()] #create three locks for synchronization +for p in range(5): #loop over the processes + Process(target=kernel.train,args=(p,lock,pool_lock)).start() #start each process with the train function and pass the process id, the number of episodes, the locks and the pool locks as arguments +kernel.visualize_reward() #visualize the reward +kernel.visualize_train() #visualize the loss +``` + ### Stop training and saving when condition is met: ```python import Note.RL.parallel.kernel as k #import kernel module From ee6f7c6d3e7cc026ec7b52e309279a0667bcbd54 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Mon, 7 Aug 2023 13:47:54 +0800 Subject: [PATCH 153/337] Update README.md --- README.md | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/README.md b/README.md index a2f375b5..d475b5d3 100644 --- a/README.md +++ b/README.md @@ -343,6 +343,33 @@ for p in range(3): #loop over the processes Process(target=kernel.train,args=(p,None,None,32)).start() #start each process with the train function and pass the process id, the locks, the test flag and the test batch size as arguments ``` +### Saving multiple files in parallel training: +```python +import Note.DL.parallel.kernel as k #import kernel module +import tensorflow as tf #import tensorflow library +import nn as n #import neural network module +from multiprocessing import Process,Manager #import multiprocessing tools +mnist=tf.keras.datasets.mnist #load mnist dataset +(x_train,y_train),(x_test,y_test)=mnist.load_data() #split data into train and test sets +x_train,x_test =x_train/255.0,x_test/255.0 #normalize data +nn=n.nn() #create neural network object +nn.build() #build the network structure +kernel=k.kernel(nn) #create kernel object with the network +kernel.process=3 #set the number of processes to train +kernel.data_segment_flag=True #set the flag to segment data for each process +kernel.epoch=5 #set the number of epochs to train +kernel.batch=32 #set the batch size +kernel.PO=3 #use PO3 algorithm for parallel optimization +kernel.s=3 #set the maximum number of saved files +kernel.data(x_train,y_train) #input train data to the kernel +manager=Manager() #create manager object to share data among processes +kernel.init(manager) #initialize shared data with the manager +for p in range(3): #loop over the processes + Process(target=kernel.train,args=(p,)).start() #start each process with the train function and pass the process id and the lock as arguments +kernel.update_nn_param() #update the network parameters after training +kernel.test(x_train,y_train,32) #test the network performance on the train set with batch size 32 +``` + ### Stop training and saving when condition is met: ```python import Note.DL.parallel.kernel as k #import kernel module From b49a5280f900fad3a50009609926686a326cb34a Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Mon, 7 Aug 2023 16:01:19 +0800 Subject: [PATCH 154/337] Update README.md --- README.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/README.md b/README.md index d475b5d3..ea0662e1 100644 --- a/README.md +++ b/README.md @@ -544,6 +544,26 @@ kernel.visualize_reward() #visualize the reward kernel.visualize_train() #visualize the loss ``` +### Saving multiple files in parallel training: +```python +import Note.RL.parallel.kernel as k #import kernel module +import DQN as d #import deep Q-network module +from multiprocessing import Process,Lock,Manager #import multiprocessing tools +dqn=d.DQN(4,128,2) #create neural network object with 4 inputs, 128 hidden units and 2 outputs +kernel=k.kernel(dqn,5) #create kernel object with the network and 5 processes to train +kernel.episode=100 #set the number of episodes to 100 +manager=Manager() #create manager object to share data among processes +kernel.init(manager) #initialize shared data with the manager +kernel.action_count=2 #set the number of actions to 2 +kernel.set_up(epsilon=0.01,pool_size=10000,batch=64,update_step=10) #set up the hyperparameters for training +kernel.PO=3 #use PO3 algorithm for parallel optimization +kernel.s=3 #set the maximum number of saved files +pool_lock=[Lock(),Lock(),Lock(),Lock(),Lock()] #create a list of locks for each process's replay pool +lock=[Lock(),Lock()] #create two locks for synchronization +for p in range(5): #loop over the processes + Process(target=kernel.train,args=(p,lock,pool_lock)).start() #start each process with the train function and pass the process id, the number of episodes, the locks and the pool locks as arguments +``` + ### Stop training and saving when condition is met: ```python import Note.RL.parallel.kernel as k #import kernel module From e74fa42e7e8a07687e6edca0293e4712139337b1 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Mon, 7 Aug 2023 19:40:21 +0800 Subject: [PATCH 155/337] Update README.md --- README.md | 87 ++++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 73 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index ea0662e1..ee3e65be 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,41 @@ kernel.restore('save.dat') #restore the network from a file kernel.train(32,1) #train the network again with batch size 32 and epoch 1 ``` +## Training with test data +**You can get neural network example from the link below, and then you can import neural network and train with kernel, example code are below.** + +https://github.com/NoteDancing/Note-documentation/blob/Note-7.0/Note%207.0%20documentation/DL/neural%20network/tensorflow/nn_acc.py + +```python +import Note.DL.kernel as k #import kernel module +import tensorflow as tf #import tensorflow library +import nn_acc as n #import neural network module with accuracy function +mnist=tf.keras.datasets.mnist #load mnist dataset +(x_train,y_train),(x_test,y_test)=mnist.load_data() #split data into train and test sets +x_train,x_test =x_train/255.0,x_test/255.0 #normalize data +nn=n.nn() #create neural network object +kernel=k.kernel(nn) #create kernel object with the network +kernel.platform=tf #set the platform to tensorflow +kernel.data(x_train,y_train,x_test,y_test) #input train and test data and labels to the kernel +kernel.train(32,5,32) #train the network with batch size 32, epoch 5 and test batch size 32 +kernel.test(x_test,y_test,32)#test the network performance on the test set with batch size 32 +``` + +### Saving multiple files in training: +```python +import Note.DL.kernel as k #import kernel module +import tensorflow as tf #import tensorflow library +import nn as n #import neural network module +mnist=tf.keras.datasets.mnist #load mnist dataset +(x_train,y_train),(x_test,y_test)=mnist.load_data() #split data into train and test sets +x_train,x_test =x_train/255.0,x_test/255.0 #normalize data +nn=n.nn() #create neural network object +kernel=k.kernel(nn) #create kernel object with the network +kernel.platform=tf #set the platform to tensorflow +kernel.data(x_train,y_train) #input train data to the kernel +kernel.train(32,5,save=True,one=False,s=3) #train the network with batch size 32 and epoch 5 +``` + ### Stop training and saving when condition is met: ```python import Note.DL.kernel as k #import kernel module @@ -63,8 +98,36 @@ kernel.train(32,5) #train the network with batch size 32 and epoch 5 kernel.visualize_train() #visualize the loss ``` +### Set the print count: +```python +import Note.DL.kernel as k #import kernel module +import tensorflow as tf #import tensorflow library +import nn as n #import neural network module +mnist=tf.keras.datasets.mnist #load mnist dataset +(x_train,y_train),(x_test,y_test)=mnist.load_data() #split data into train and test sets +x_train,x_test =x_train/255.0,x_test/255.0 #normalize data +nn=n.nn() #create neural network object +kernel=k.kernel(nn) #create kernel object with the network +kernel.platform=tf #set the platform to tensorflow +kernel.data(x_train,y_train) #input train data to the kernel +kernel.train(32,5,p=3) #train the network with batch size 32 and epoch 5 +``` + ## RL: +### Saving multiple files in training: +```python +import Note.RL.kernel as k #import kernel module +import tensorflow as tf #import tensorflow library +import DQN as d #import deep Q-network module +dqn=d.DQN(4,128,2) #create neural network object with 4 inputs, 128 hidden units and 2 outputs +kernel=k.kernel(dqn) #create kernel object with the network +kernel.platform=tf #set the platform to tensorflow +kernel.action_count=2 #set the number of actions to 2 +kernel.set_up(epsilon=0.01,pool_size=10000,batch=64,update_step=10) #set up the hyperparameters for training +kernel.train(100,save=True,one=False,s=3) #train the network for 100 episodes +``` + ### Stop training and saving when condition is met: ```python import Note.RL.kernel as k #import kernel module @@ -91,23 +154,19 @@ kernel.visualize_reward() #visualize the reward kernel.visualize_train() #visualize the loss ``` -## Training with test data -https://github.com/NoteDancing/Note-documentation/blob/Note-7.0/Note%207.0%20documentation/DL/neural%20network/tensorflow/nn_acc.py +### Set the print count: ```python -import Note.DL.kernel as k #import kernel module -import tensorflow as tf #import tensorflow library -import nn_acc as n #import neural network module with accuracy function -mnist=tf.keras.datasets.mnist #load mnist dataset -(x_train,y_train),(x_test,y_test)=mnist.load_data() #split data into train and test sets -x_train,x_test =x_train/255.0,x_test/255.0 #normalize data -nn=n.nn() #create neural network object -kernel=k.kernel(nn) #create kernel object with the network -kernel.platform=tf #set the platform to tensorflow -kernel.data(x_train,y_train,x_test,y_test) #input train and test data and labels to the kernel -kernel.train(32,5,32) #train the network with batch size 32, epoch 5 and test batch size 32 -kernel.test(x_test,y_test,32)#test the network performance on the test set with batch size 32 +import Note.RL.kernel as k #import kernel module +import DQN as d #import deep Q-network module +dqn=d.DQN(4,128,2) #create neural network object with 4 inputs, 128 hidden units and 2 outputs +kernel=k.kernel(dqn) #create kernel object with the network +kernel.stop=True #set the flag to stop training when a condition is met +kernel.action_count=2 #set the number of actions to 2 +kernel.set_up(epsilon=0.01,pool_size=10000,batch=64,update_step=10,trial_count=10,criterion=200) #set up the hyperparameters for training and the condition to stop training when the average reward of 10 trials is greater than 200 +kernel.train(100,p=3) #train the network for 500 episodes ``` + ## Parallel test: **You can get neural network example from the link below, and then you can import neural network and train with kernel, example code are below.** From c8f29fb070d0d0256666d7bb5b96eff8630c3fbc Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Mon, 7 Aug 2023 20:49:13 +0800 Subject: [PATCH 156/337] Update README.md --- README.md | 5 ----- 1 file changed, 5 deletions(-) diff --git a/README.md b/README.md index ee3e65be..57f30d44 100644 --- a/README.md +++ b/README.md @@ -519,11 +519,6 @@ kernel.test(x_train,y_train,32) #test the network performance on the train ## RL： **Pool Network:** - -**You can get neural network example from the link below, and then you can import neural network and train with kernel, example code are below.** - -https://github.com/NoteDancing/Note-documentation/blob/Note-7.0/Note%207.0%20documentation/RL/neural%20network/tensorflow/parallel/DQN.py - ### PO1: ```python import Note.RL.parallel.kernel as k #import kernel module From 3adb2f8d296c0aa4e85bde6b3af29c73ca112da2 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Tue, 8 Aug 2023 22:30:43 +0800 Subject: [PATCH 157/337] Update kernel.py --- .../DL/kernel/parallel/kernel.py | 30 ++++++++----------- 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/Note 7.0 documentation/DL/kernel/parallel/kernel.py b/Note 7.0 documentation/DL/kernel/parallel/kernel.py index 9e81b4e2..7eae0132 100644 --- a/Note 7.0 documentation/DL/kernel/parallel/kernel.py +++ b/Note 7.0 documentation/DL/kernel/parallel/kernel.py @@ -75,24 +75,18 @@ def data(self,train_data=None,train_labels=None,test_data=None,test_labels=None, def segment_data(self): # a method to segment the data for each process - if len(self.train_data)!=self.process: - segments=int((len(self.train_data)-len(self.train_data)%self.process)/self.process) # calculate how many segments to divide the data into - for i in range(self.process): - index1=i*segments # get the start index of each segment - index2=(i+1)*segments # get the end index of each segment - if i==0: - data=np.expand_dims(self.train_data[index1:index2],axis=0) # create a new axis for each segment of data - labels=np.expand_dims(self.train_labels[index1:index2],axis=0) # create a new axis for each segment of labels - else: - data=np.concatenate((data,np.expand_dims(self.train_data[index1:index2],axis=0))) # concatenate the segments of data along the new axis - labels=np.concatenate((labels,np.expand_dims(self.train_labels[index1:index2],axis=0))) # concatenate the segments of labels along the new axis - if len(data)%self.process!=0: # if there are some remaining samples - segments+=1 # add one more segment - index1=segments*self.process # get the start index of the last segment - index2=self.process-(len(self.train_data)-segments*self.process) # get the end index of the last segment - data=np.concatenate((data,np.expand_dims(self.train_data[index1:index2],axis=0))) # concatenate the last segment of data along the new axis - labels=np.concatenate((labels,np.expand_dims(self.train_labels[index1:index2],axis=0))) # concatenate the last segment of labels along the new axis - return data,labels # return the segmented data and labels + # calculate the number of data to be sliced + length = len(self.train_data) - len(self.train_data) % self.process + # slice the front part of the data and labels + data = self.train_data[:length] + labels = self.train_labels[:length] + # split the data and labels into subarrays + data = np.split(data, self.process) + labels = np.split(labels, self.process) + # stack the data and labels along a new axis + data = np.stack(data, axis=0) + labels = np.stack(labels, axis=0) + return data, labels def init(self,manager): From be4f83eafef8641c6a6efcf3a55639c0828385b4 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Tue, 8 Aug 2023 22:31:53 +0800 Subject: [PATCH 158/337] Update kernel.py --- .../DL/kernel/parallel/kernel.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Note 7.0 documentation/DL/kernel/parallel/kernel.py b/Note 7.0 documentation/DL/kernel/parallel/kernel.py index 7eae0132..5615031b 100644 --- a/Note 7.0 documentation/DL/kernel/parallel/kernel.py +++ b/Note 7.0 documentation/DL/kernel/parallel/kernel.py @@ -76,17 +76,17 @@ def data(self,train_data=None,train_labels=None,test_data=None,test_labels=None, def segment_data(self): # a method to segment the data for each process # calculate the number of data to be sliced - length = len(self.train_data) - len(self.train_data) % self.process + length=len(self.train_data)-len(self.train_data)%self.process # slice the front part of the data and labels - data = self.train_data[:length] - labels = self.train_labels[:length] + data=self.train_data[:length] + labels=self.train_labels[:length] # split the data and labels into subarrays - data = np.split(data, self.process) - labels = np.split(labels, self.process) + data=np.split(data, self.process) + labels=np.split(labels, self.process) # stack the data and labels along a new axis - data = np.stack(data, axis=0) - labels = np.stack(labels, axis=0) - return data, labels + data=np.stack(data, axis=0) + labels=np.stack(labels, axis=0) + return data,labels def init(self,manager): From b6a76830e5e1ac12789d13674cafcff0c5baef90 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 10 Aug 2023 19:16:33 +0800 Subject: [PATCH 159/337] Update kernel.py --- Note 7.0 documentation/DL/kernel/parallel/kernel.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/Note 7.0 documentation/DL/kernel/parallel/kernel.py b/Note 7.0 documentation/DL/kernel/parallel/kernel.py index 5615031b..02adce7f 100644 --- a/Note 7.0 documentation/DL/kernel/parallel/kernel.py +++ b/Note 7.0 documentation/DL/kernel/parallel/kernel.py @@ -83,9 +83,6 @@ def segment_data(self): # split the data and labels into subarrays data=np.split(data, self.process) labels=np.split(labels, self.process) - # stack the data and labels along a new axis - data=np.stack(data, axis=0) - labels=np.stack(labels, axis=0) return data,labels From 51fccaced4447da9f4da5b51d924fc7186d20476 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Sat, 12 Aug 2023 13:57:29 +0800 Subject: [PATCH 160/337] Update kernel_pytorch.py --- .../DL/kernel/parallel/kernel_pytorch.py | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/Note 7.0 documentation/DL/kernel/parallel/kernel_pytorch.py b/Note 7.0 documentation/DL/kernel/parallel/kernel_pytorch.py index 6958343c..b3263de1 100644 --- a/Note 7.0 documentation/DL/kernel/parallel/kernel_pytorch.py +++ b/Note 7.0 documentation/DL/kernel/parallel/kernel_pytorch.py @@ -122,7 +122,7 @@ def init_online(self,manager): return # return nothing - def end(self): + def end(self): '''define a method named end that takes no arguments''' '''This method is used to check if the training process should be stopped based on some criteria''' @@ -133,20 +133,20 @@ def end(self): '''The method will return a boolean value indicating whether the training process should be stopped or not''' - if self.end_loss!=None and len(self.train_loss_list)!=0 and self.train_loss_list[-1]self.end_acc: # check if the end_acc attribute is not None and the train_acc_list attribute is not empty and the last element of the train_acc_list attribute is greater than the end_acc attribute - return True # return True - elif self.end_loss!=None and len(self.train_loss_list)!=0 and self.end_acc!=None and self.train_loss_list[-1]self.end_acc: # check if both previous conditions are met - return True # return True - elif self.end_test_loss!=None and len(self.test_loss_list)!=0 and self.test_loss_list[-1]self.end_test_acc: # check if the end_test_acc attribute is not None and the test_acc_list attribute is not empty and the last element of the test_acc_list attribute is greater than the end_test_acc attribute - return True # return True - elif self.end_test_loss!=None and len(self.test_loss_list)!=0 and self.end_test_acc!=None and self.test_loss_list[-1]self.end_test_acc: # check if both previous conditions are met - return True # return True - else: # otherwise - return False # return False + if self.end_acc!=None and len(self.train_acc_list)!=0 and self.train_acc_list[-1]>self.end_acc: + return True + elif self.end_loss!=None and len(self.train_loss_list)!=0 and self.train_loss_list[-1]self.end_test_acc: + return True + elif self.end_test_loss!=None and len(self.test_loss_list)!=0 and self.test_loss_list[-1]self.end_acc and self.test_acc_list[-1]>self.end_test_acc: + return True + elif self.end_loss!=None and self.end_test_loss!=None: + if len(self.train_loss_list)!=0 and len(self.test_loss_list)!=0 and self.train_loss_list[-1] Date: Sat, 12 Aug 2023 13:57:41 +0800 Subject: [PATCH 161/337] Update kernel.py --- .../DL/kernel/parallel/kernel.py | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/Note 7.0 documentation/DL/kernel/parallel/kernel.py b/Note 7.0 documentation/DL/kernel/parallel/kernel.py index 02adce7f..64b77734 100644 --- a/Note 7.0 documentation/DL/kernel/parallel/kernel.py +++ b/Note 7.0 documentation/DL/kernel/parallel/kernel.py @@ -142,18 +142,20 @@ def init_online(self,manager): def end(self): # a method to check whether to end the training according to some conditions - if self.end_loss!=None and len(self.train_loss_list)!=0 and self.train_loss_list[-1]self.end_acc: - return True # return True if the training accuracy is higher than the end accuracy - elif self.end_loss!=None and len(self.train_loss_list)!=0 and self.end_acc!=None and self.train_loss_list[-1]self.end_acc: - return True # return True if both the training loss and accuracy meet the end conditions - elif self.end_test_loss!=None and len(self.test_loss_list)!=0 and self.test_loss_list[-1]self.end_acc: + return True + elif self.end_loss!=None and len(self.train_loss_list)!=0 and self.train_loss_list[-1]self.end_test_acc: - return True # return True if the testing accuracy is higher than the end test accuracy - elif self.end_test_loss!=None and len(self.test_loss_list)!=0 and self.end_test_acc!=None and self.test_loss_list[-1]self.end_test_acc: - return True # return True if both the testing loss and accuracy meet the end conditions + return True + elif self.end_test_loss!=None and len(self.test_loss_list)!=0 and self.test_loss_list[-1]self.end_acc and self.test_acc_list[-1]>self.end_test_acc: + return True + elif self.end_loss!=None and self.end_test_loss!=None: + if len(self.train_loss_list)!=0 and len(self.test_loss_list)!=0 and self.train_loss_list[-1] Date: Sat, 12 Aug 2023 13:57:53 +0800 Subject: [PATCH 162/337] Update kernel.py --- Note 7.0 documentation/DL/kernel/kernel.py | 30 ++++++++++++---------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/Note 7.0 documentation/DL/kernel/kernel.py b/Note 7.0 documentation/DL/kernel/kernel.py index 92ecad36..81c0d91b 100644 --- a/Note 7.0 documentation/DL/kernel/kernel.py +++ b/Note 7.0 documentation/DL/kernel/kernel.py @@ -88,20 +88,22 @@ def init(self): return # return from the method - def end(self): - # define a method for checking if some conditions are met for ending training or testing process - if self.end_loss!=None and len(self.train_loss_list)!=0 and self.train_loss_list[-1]self.end_acc: # check if the end_acc attribute is not None and the train_acc_list attribute is not empty and the last element of the train_acc_list attribute is greater than the end_acc attribute - return True # return True - elif self.end_loss!=None and len(self.train_loss_list)!=0 and self.end_acc!=None and self.train_loss_list[-1]self.end_acc: # check if the end_loss attribute and the end_acc attribute are not None and the train_loss_list attribute and the train_acc_list attribute are not empty and the last element of the train_loss_list attribute is less than the end_loss attribute and the last element of the train_acc_list attribute is greater than the end_acc attribute - return True # return True - elif self.end_test_loss!=None and len(self.test_loss_list)!=0 and self.test_loss_list[-1]self.end_test_acc: # check if the end_test_acc attribute is not None and the test_acc_list attribute is not empty and the last element of the test_acc_list attribute is greater than the end_test_acc attribute - return True # return True - elif self.end_test_loss!=None and len(self.test_loss_list)!=0 and self.end_test_acc!=None and self.test_loss_list[-1]self.end_test_acc: # check if the end_test_loss attribute and the end_test_acc attribute are not None and the test_loss_list attribute and the test_acc_list attribute are not empty and the last element of the test_loss_list attribute is less than the end_test_loss attribute and the last element of the test_acc_list attribute is greater than the end_test_acc attribute - return True # return True + def end(self): + # define a method for checking if some conditions are met for ending training + if self.end_acc!=None and self.train_acc!=None and self.train_acc>self.end_acc: + return True + elif self.end_loss!=None and self.train_loss!=None and self.train_lossself.end_test_acc: + return True + elif self.end_test_loss!=None and self.test_loss!=None and self.test_lossself.end_acc and self.test_acc>self.end_test_acc: + return True + elif self.end_loss!=None and self.end_test_loss!=None: + if self.train_loss!=None and self.test_loss!=None and self.train_loss Date: Sat, 12 Aug 2023 22:22:18 +0800 Subject: [PATCH 163/337] Add files via upload --- Note 7.0 documentation/DL/dl/test.py | 216 +++++++++++++++++++++++++++ 1 file changed, 216 insertions(+) create mode 100644 Note 7.0 documentation/DL/dl/test.py diff --git a/Note 7.0 documentation/DL/dl/test.py b/Note 7.0 documentation/DL/dl/test.py new file mode 100644 index 00000000..1f07ec09 --- /dev/null +++ b/Note 7.0 documentation/DL/dl/test.py @@ -0,0 +1,216 @@ +import tensorflow as tf # import TensorFlow library +import numpy as np # import NumPy library +from multiprocessing import Array # import Array class from multiprocessing module +import numpy.ctypeslib as npc # import ctypeslib module from NumPy + + +@tf.function(jit_compile=True) # use TensorFlow's just-in-time compilation decorator +def test_tf(nn,data,labels): # define a function that tests a TensorFlow network + try: # try to execute the following block of code + try: # try to execute the following block of code + output=nn.fp(data) # get the output of the network by calling its forward propagation method + loss=nn.loss(output,labels) # calculate the loss by calling its loss function + except Exception: # if an exception occurs in the previous block + output,loss=nn.fp(data,labels) # get the output and loss of the network by calling its forward propagation method with both data and labels as arguments + except Exception as e: # if an exception occurs in the previous block + raise e # re-raise the exception + try: # try to execute the following block of code + if hasattr(nn,'accuracy'): # check if the network has an accuracy attribute + acc=nn.accuracy(output,labels) # calculate the accuracy by calling its accuracy function + else: # if the network does not have an accuracy attribute + acc=None # set the accuracy to None + except Exception as e: # if an exception occurs in the previous block + raise e # re-raise the exception + return loss,acc # return the loss and accuracy + + +def test_pytorch(nn,data,labels): # define a function that tests a PyTorch network + try: # try to execute the following block of code + try: # try to execute the following block of code + output=nn.fp(data) # get the output of the network by calling its forward propagation method + loss=nn.loss(output,labels) # calculate the loss by calling its loss function + except Exception: # if an exception occurs in the previous block + output,loss=nn.fp(data,labels) # get the output and loss of the network by calling its forward propagation method with both data and labels as arguments + except Exception as e: # if an exception occurs in the previous block + raise e # re-raise the exception + try: # try to execute the following block of code + if hasattr(nn,'accuracy'): # check if the network has an accuracy attribute + acc=nn.accuracy(output,labels) # calculate the accuracy by calling its accuracy function + else: # if the network does not have an accuracy attribute + acc=None # set the accuracy to None + except Exception as e: # if an exception occurs in the previous block + raise e # re-raise the exception + return loss,acc # return the loss and accuracy + + +def test(nn,test_data,test_labels,platform,batch=None,loss=None,acc_flag='%'): # define a function that tests a generic network on a given platform + if type(nn.param[0])!=list: # check if the first element of nn.param is not a list + test_data=test_data.astype(nn.param[0].dtype.name) # convert test_data to match the data type of nn.param[0] + test_labels=test_labels.astype(nn.param[0].dtype.name) # convert test_labels to match the data type of nn.param[0] + else: # if the first element of nn.param is a list + test_data=test_data.astype(nn.param[0][0].dtype.name) # convert test_data to match the data type of nn.param[0][0] + test_labels=test_labels.astype(nn.param[0][0].dtype.name) # convert test_labels to match the data type of nn.param[0][0] + if batch!=None: # check if batch is not None + total_loss=0 # initialize total_loss to zero + total_acc=0 # initialize total_acc to zero + batches=int((test_data.shape[0]-test_data.shape[0]%batch)/batch) # calculate how many batches are needed + shape0=test_data.shape[0] # store the original shape of test_data along axis 0 + for j in range(batches): # loop through each batch + index1=j*batch # calculate the starting index of data_batch + index2=(j+1)*batch # calculate the ending index of data_batch + data_batch=test_data[index1:index2] # slice test_data to get data_batch + labels_batch=test_labels[index1:index2] # slice test_labels to get labels_batch + if hasattr(platform,'DType'): # check if platform has a DType attribute (indicating TensorFlow) + batch_loss,batch_acc=test_tf(data_batch,labels_batch) # call test_tf function to get batch_loss and batch_acc + else: # if platform does not have a DType attribute (indicating PyTorch) + batch_loss,batch_acc=test_pytorch(data_batch,labels_batch) # call test_pytorch function to get batch_loss and batch_acc + total_loss+=batch_loss # add batch_loss to total_loss + if hasattr(nn,'accuracy'): # check if the network has an accuracy attribute + total_acc+=batch_acc # add batch_acc to total_acc + if shape0%batch!=0: # check if there is a remainder after dividing test_data by batch + batches+=1 # increment batches by one + index1=batches*batch # calculate the starting index of data_batch + index2=batch-(shape0-batches*batch) # calculate the ending index of data_batch + try: # try to execute the following block of code + try: # try to execute the following block of code + data_batch=platform.concat([test_data[index1:],test_data[:index2]],0) # concatenate the remaining test_data and some test_data from the beginning along axis 0 to get data_batch + labels_batch=platform.concat([test_labels[index1:],test_labels[:index2]],0) # concatenate the corresponding labels along axis 0 to get labels_batch + except Exception: # if an exception occurs in the previous block + data_batch=np.concatenate([test_data[index1:],test_data[:index2]],0) # use NumPy's concatenate function instead of platform's concat function + labels_batch=np.concatenate([test_labels[index1:],test_labels[:index2]],0) # use NumPy's concatenate function instead of platform's concat function + except Exception as e: # if an exception occurs in the previous block + raise e # re-raise the exception + if hasattr(platform,'DType'): # check if platform has a DType attribute (indicating TensorFlow) + batch_loss,batch_acc=test_tf(data_batch,labels_batch) # call test_tf function to get batch_loss and batch_acc + else: # if platform does not have a DType attribute (indicating PyTorch) + batch_loss,batch_acc=test_pytorch(data_batch,labels_batch) # call test_pytorch function to get batch_loss and batch_acc + total_loss+=batch_loss # add batch_loss to total_loss + if hasattr(nn,'accuracy'): # check if the network has an accuracy attribute + total_acc+=batch_acc # add batch_acc to total_acc + test_loss=total_loss.numpy()/batches # calculate the average test loss by dividing total_loss by batches and converting it to a NumPy array + test_loss=test_loss.astype(np.float32) # convert test_loss to float32 type + if hasattr(nn,'accuracy'): # check if the network has an accuracy attribute + test_acc=total_acc.numpy()/batches # calculate the average test accuracy by dividing total_acc by batches and converting it to a NumPy array + test_acc=test_acc.astype(np.float32) # convert test_acc to float32 type + else: # if batch is None + if hasattr(platform,'DType'): # check if platform has a DType attribute (indicating TensorFlow) + batch_loss,batch_acc=test_tf(test_data,test_labels) # call test_tf function to get batch_loss and batch_acc + else: # if platform does not have a DType attribute (indicating PyTorch) + batch_loss,batch_acc=test_pytorch(test_data,test_labels) # call test_pytorch function to get batch_loss and batch_acc + test_loss=batch_loss.numpy().astype(np.float32) # convert batch_loss to a NumPy array and float32 type + if hasattr(nn,'accuracy'): # check if the network has an accuracy attribute + test_acc=batch_acc.numpy().astype(np.float32) # convert batch_acc to a NumPy array and float32 type + print('test loss:{0:.6f}'.format(test_loss)) # print the test loss with six decimal places + if hasattr(nn,'accuracy'): # check if the network has an accuracy attribute + if acc_flag=='%': # check if acc_flag is '%' (indicating percentage format) + print('test acc:{0:.1f}'.format(test_acc*100)) # print the test accuracy with one decimal place and multiply by 100 + else: # if acc_flag is not '%' (indicating decimal format) + print('test acc:{0:.6f}'.format(test_acc)) # print the test accuracy with six decimal places + if acc_flag=='%': # check if acc_flag is '%' (indicating percentage format) + return test_loss,test_acc*100 # return the test loss and accuracy multiplied by 100 + else: # if acc_flag is not '%' (indicating decimal format) + return test_loss,test_acc # return the test loss and accuracy + else: # if the network does not have an accuracy attribute + return test_loss # return the test loss + + +class parallel_test: # define a class for parallel testing + def __init__(self,nn,test_data,test_labels,process,batch,prefetch_batch_size=tf.data.AUTOTUNE,test_dataset=None,): # define the constructor method + self.nn=nn # store the neural network as an attribute + if test_data is not None and type(self.nn.param[0])!=list: # check if test_data is not None and the first element of nn.param is not a list + self.test_data=test_data.astype(self.nn.param[0].dtype.name) # convert test_data to match the data type of nn.param[0] + self.test_labels=test_labels.astype(self.nn.param[0].dtype.name) # convert test_labels to match the data type of nn.param[0] + elif test_data is not None: # if test_data is not None but the first element of nn.param is a list + self.test_data=test_data.astype(self.nn.param[0][0].dtype.name) # convert test_data to match the data type of nn.param[0][0] + self.test_labels=test_labels.astype(self.nn.param[0][0].dtype.name) # convert test_labels to match the data type of nn.param[0][0] + self.test_dataset=test_dataset # store the test_dataset as an attribute + self.process=process # store the number of processes as an attribute + self.batch=batch # store the batch size as an attribute + if type(self.nn.param[0])!=list: # check if the first element of nn.param is not a list + self.loss=Array('f',np.zeros([process],dtype=self.nn.param[0].dtype.name)) # create a shared memory array for storing the loss values for each process with the same data type as nn.param[0] + else: # if the first element of nn.param is a list + self.loss=Array('f',np.zeros([process],dtype=self.nn.param[0][0].dtype.name)) # create a shared memory array for storing the loss values for each process with the same data type as nn.param[0][0] + if hasattr(nn,'accuracy'): # check if the network has an accuracy attribute + if type(self.nn.param[0])!=list: # check if the first element of nn.param is not a list + self.acc=Array('f',np.zeros([process],dtype=self.nn.param[0].dtype.name)) # create a shared memory array for storing the accuracy values for each process with the same data type as nn.param[0] + else: # if the first element of nn.param is a list + self.acc=Array('f',np.zeros([process],dtype=self.nn.param[0][0].dtype.name)) # create a shared memory array for storing the accuracy values for each process with the same data type as nn.param[0][0] + self.prefetch_batch_size=prefetch_batch_size # store the prefetch batch size as an attribute + + + def segment_data(self): # define a method that segments the test data and labels into equal parts for each process + if len(self.test_data)!=self.process: # check if the length of test_data is not equal to the number of processes + length=len(self.test_data)-len(self.test_data)%self.process # calculate the length of test_data that can be evenly divided by the number of processes + data=self.test_data[:length] # slice test_data to get only that part + labels=self.test_labels[:length] # slice test_labels to get only that part + data=np.split(data,self.process) # split data into equal parts for each process + labels=np.split(labels,self.process) # split labels into equal parts for each process + self.test_data=data # assign data to test_data attribute + self.test_labels=labels # assign labels to test_labels attribute + return + + + @tf.function(jit_compile=True) # use TensorFlow's just-in-time compilation decorator + def test_(self,data,labels,p): # define a method that tests a TensorFlow network on a given data, labels, and process index + try: # try to execute the following block of code + try: # try to execute the following block of code + try: # try to execute the following block of code + output=self.nn.fp(data,p) # get the output of the network by calling its forward propagation method with data and process index as arguments + loss=self.nn.loss(output,labels,p) # calculate the loss by calling its loss function with output, labels, and process index as arguments + except Exception: # if an exception occurs in the previous block + output,loss=self.nn.fp(data,labels,p) # get the output and loss of the network by calling its forward propagation method with data, labels, and process index as arguments + except Exception: # if an exception occurs in the previous block + try: # try to execute the following block of code + output=self.nn.fp(data) # get the output of the network by calling its forward propagation method with data as argument + loss=self.nn.loss(output,labels) # calculate the loss by calling its loss function with output and labels as arguments + except Exception: # if an exception occurs in the previous block + output,loss=self.nn.fp(data,labels) # get the output and loss of the network by calling its forward propagation method with data and labels as arguments + except Exception as e: # if an exception occurs in the previous block + raise e # re-raise the exception + try: # try to execute the following block of code + if hasattr(self.nn,'accuracy'): # check if the network has an accuracy attribute + try: # try to execute the following block of code + acc=self.nn.accuracy(output,labels,p) # calculate the accuracy by calling its accuracy function with output, labels, and process index as arguments + except Exception: # if an exception occurs in the previous block + acc=self.nn.accuracy(output,labels) # calculate the accuracy by calling its accuracy function with output and labels as arguments + else: # if the network does not have an accuracy attribute + acc=None # set the accuracy to None + except Exception as e: # if an exception occurs in the previous block + raise e # re-raise the exception + return loss,acc # return the loss and accuracy + + + def test(self,p): # define a method that tests a generic network on a given process index + if self.test_dataset is None: # check if test_dataset is None + test_ds=tf.data.Dataset.from_tensor_slices((self.test_data[p],self.test_labels[p])).batch(self.batch).prefetch(self.prefetch_batch_size) # create a TensorFlow dataset from test_data and test_labels for the given process index, batch it, and prefetch it + elif self.test_dataset is not None and type(self.test_dataset)==list: # check if test_dataset is not None and is a list + test_ds=self.test_dataset[p] # get the test_dataset for the given process index + else: # if test_dataset is not None and is not a list + test_ds=self.test_dataset # use test_dataset as it is + for data_batch,labels_batch in test_ds: # loop through each batch of data and labels in test_ds + try: # try to execute the following block of code + batch_loss,batch_acc=self.test_(data_batch,labels_batch,p) # call test_ method to get batch_loss and batch_acc + except Exception as e: # if an exception occurs in the previous block + raise e # re-raise the exception + if hasattr(self.nn,'accuracy'): # check if the network has an accuracy attribute + self.loss[p]+=batch_loss # add batch_loss to loss array for the given process index + self.acc[p]+=batch_acc # add batch_acc to acc array for the given process index + else: # if the network does not have an accuracy attribute + self.loss[p]+=batch_loss # add batch_loss to loss array for the given process index + return + + + def loss_acc(self): # define a method that calculates and returns the average loss and accuracy across all processes + if self.test_dataset is None: # check if test_dataset is None + shape=len(self.test_data[0])*self.process # calculate the total number of data points by multiplying the length of test_data for one process by the number of processes + elif self.test_dataset is not None and type(self.test_dataset)==list: # check if test_dataset is not None and is a list + shape=len(self.test_dataset[0])*len(self.test_dataset) # calculate the total number of data points by multiplying the length of test_dataset for one process by the length of test_dataset list + else: # if test_dataset is not None and is not a list + shape=len(self.test_dataset)*self.process # calculate the total number of data points by multiplying the length of test_dataset by the number of processes + batches=int((shape-shape%self.batch)/self.batch) # calculate how many batches are needed by dividing shape by batch size and rounding down + if shape%self.batch!=0: # check if there is a remainder after dividing shape by batch size + batches+=1 # increment batches by one + if hasattr(self.nn,'accuracy'): # check if the network has an accuracy attribute + return np.sum(npc.as_array(self.loss.get_obj()))/batches,np.sum(npc.as_array(self.acc.get_obj()))/batches # return the average loss and accuracy by summing up the loss and acc arrays and dividing them by batches + else: # if the network does not have an accuracy attribute + return np.sum(npc.as_array(self.loss.get_obj()))/batches # return the average loss by summing up the loss array and dividing it by batches \ No newline at end of file From 52033520865b0ea3a8b01a4e7d90ed298702b7fd Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Sun, 13 Aug 2023 00:23:19 +0800 Subject: [PATCH 164/337] Update README.md --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 57f30d44..ba67b80c 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ kernel.train(32,1) #train the network again with batch size 32 and ep ``` ## Training with test data -**You can get neural network example from the link below, and then you can import neural network and train with kernel, example code are below.** +**You can get a neural network example from the link below, and then you can import neural network and train with kernel, example code are below.** https://github.com/NoteDancing/Note-documentation/blob/Note-7.0/Note%207.0%20documentation/DL/neural%20network/tensorflow/nn_acc.py @@ -168,7 +168,7 @@ kernel.train(100,p=3) #train the network for 500 episodes ## Parallel test: -**You can get neural network example from the link below, and then you can import neural network and train with kernel, example code are below.** +**You can get a neural network example from the link below, and then you can import neural network and train with kernel, example code are below.** https://github.com/NoteDancing/Note-documentation/blob/Note-7.0/Note%207.0%20documentation/DL/neural%20network/tensorflow/parallel/nn.py @@ -487,7 +487,7 @@ kernel.test(x_train,y_train,32) #test the network performance on the train ### Gradient attenuation: **Calculate the attenuation coefficient based on the optimization counter using the attenuation function.** -**You can get neural network example from the link below, and then you can import neural network and train with kernel, example code are below.** +**You can get a neural network example from the link below, and then you can import neural network and train with kernel, example code are below.** https://github.com/NoteDancing/Note-documentation/blob/Note-7.0/Note%207.0%20documentation/DL/neural%20network/tensorflow/parallel/nn_attenuate.py ```python @@ -660,7 +660,7 @@ for p in range(5): #loop over the processes # Parallel test: -**You can get neural network example from the link below, and then you can import neural network and train with kernel, example code are below.** +**You can get a neural network example from the link below, and then you can import neural network and train with kernel, example code are below.** https://github.com/NoteDancing/Note-documentation/blob/Note-7.0/Note%207.0%20documentation/DL/neural%20network/tensorflow/parallel/nn.py @@ -683,7 +683,7 @@ loss,acc=test.loss_acc() #calculate the loss and accuracy of the test # Online training: -**You can get neural network example from the link below, and then you can import neural network and train with kernel, example code are below.** +**You can get a neural network example from the link below, and then you can import neural network and train with kernel, example code are below.** https://github.com/NoteDancing/Note-documentation/blob/Note-7.0/Note%207.0%20documentation/DL/neural%20network/tensorflow/non-parallel/nn_ol.py From 4d057e237a0efdd72c94f3826e37d1eb86492443 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Sun, 13 Aug 2023 09:46:24 +0800 Subject: [PATCH 165/337] Update README.md --- README.md | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/README.md b/README.md index ba67b80c..433fd6b5 100644 --- a/README.md +++ b/README.md @@ -703,6 +703,43 @@ kernel.train_online() #train the network online ``` +# Write neural network class in the interpreter: +```python +import Note.DL.kernel as k #import kernel module +import tensorflow as tf #import tensorflow library + +class nn: + def __init__(self): + self.model=tf.keras.models.Sequential([ + tf.keras.layers.Flatten(input_shape=(28, 28)), + tf.keras.layers.Dense(128,activation='relu'), + tf.keras.layers.Dropout(0.2), + tf.keras.layers.Dense(10) + ]) + self.param=self.model.weights #parameter list,kernel uses it list for backpropagation. + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.opt=tf.keras.optimizers.Adam() #optimizer,kernel uses it to optimize. + + + def fp(self,data): #forward propagation function,kernel uses it for forward propagation. + output=self.model(data) + return output + + + def loss(self,output,labels): #loss functino,kernel uses it to calculate loss. + return self.loss_object(labels,output) + +mnist=tf.keras.datasets.mnist #load mnist dataset +(x_train,y_train),(x_test,y_test)=mnist.load_data() #split data into train and test sets +x_train,x_test =x_train/255.0,x_test/255.0 #normalize data +nn=nn() #create neural network object +kernel=k.kernel(nn) #create kernel object with the network +kernel.platform=tf #set the platform to tensorflow +kernel.data(x_train,y_train) #input train data to the kernel +kernel.train(32,5) #train the network with batch size 32 and epoch 5 +``` + + # Check neural network: ## DL: You can test it before using the kernel training neural network. From 3b269b2e73b234b347934777f233bbf02a83c5f7 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Sun, 13 Aug 2023 13:55:44 +0800 Subject: [PATCH 166/337] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 433fd6b5..3481cab3 100644 --- a/README.md +++ b/README.md @@ -766,6 +766,6 @@ check(dqn,tf,2) #check the network with tensorflow platfo # GPT: -**Layers in the GPT directory created by GPT** +**Layers in the Note.nn.layer.GPT package created by GPT** https://github.com/NoteDancing/Note/tree/Note-7.0/Note/nn/layer/GPT From 655ff13311aa702f2cc5c62ff2fc6943f2a3a29d Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Mon, 14 Aug 2023 13:49:56 +0800 Subject: [PATCH 167/337] Update kernel.py --- Note 7.0 documentation/DL/kernel/parallel/kernel.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/Note 7.0 documentation/DL/kernel/parallel/kernel.py b/Note 7.0 documentation/DL/kernel/parallel/kernel.py index 64b77734..7fa9a532 100644 --- a/Note 7.0 documentation/DL/kernel/parallel/kernel.py +++ b/Note 7.0 documentation/DL/kernel/parallel/kernel.py @@ -75,14 +75,9 @@ def data(self,train_data=None,train_labels=None,test_data=None,test_labels=None, def segment_data(self): # a method to segment the data for each process - # calculate the number of data to be sliced - length=len(self.train_data)-len(self.train_data)%self.process - # slice the front part of the data and labels - data=self.train_data[:length] - labels=self.train_labels[:length] # split the data and labels into subarrays - data=np.split(data, self.process) - labels=np.split(labels, self.process) + data=np.split(self.train_data,self.process) + labels=np.split(self.train_labels,self.process) return data,labels From 272759b0451521fcbbff2b3c178eaaa1d5845134 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Mon, 14 Aug 2023 13:50:12 +0800 Subject: [PATCH 168/337] Update test.py --- Note 7.0 documentation/DL/dl/test.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/Note 7.0 documentation/DL/dl/test.py b/Note 7.0 documentation/DL/dl/test.py index 1f07ec09..13f875a8 100644 --- a/Note 7.0 documentation/DL/dl/test.py +++ b/Note 7.0 documentation/DL/dl/test.py @@ -140,11 +140,8 @@ def __init__(self,nn,test_data,test_labels,process,batch,prefetch_batch_size=tf. def segment_data(self): # define a method that segments the test data and labels into equal parts for each process if len(self.test_data)!=self.process: # check if the length of test_data is not equal to the number of processes - length=len(self.test_data)-len(self.test_data)%self.process # calculate the length of test_data that can be evenly divided by the number of processes - data=self.test_data[:length] # slice test_data to get only that part - labels=self.test_labels[:length] # slice test_labels to get only that part - data=np.split(data,self.process) # split data into equal parts for each process - labels=np.split(labels,self.process) # split labels into equal parts for each process + data=np.split(self.test_data,self.process) # split data into equal parts for each process + labels=np.split(self.test_labels,self.process) # split labels into equal parts for each process self.test_data=data # assign data to test_data attribute self.test_labels=labels # assign labels to test_labels attribute return @@ -213,4 +210,4 @@ def loss_acc(self): # define a method that calculates and returns the average lo if hasattr(self.nn,'accuracy'): # check if the network has an accuracy attribute return np.sum(npc.as_array(self.loss.get_obj()))/batches,np.sum(npc.as_array(self.acc.get_obj()))/batches # return the average loss and accuracy by summing up the loss and acc arrays and dividing them by batches else: # if the network does not have an accuracy attribute - return np.sum(npc.as_array(self.loss.get_obj()))/batches # return the average loss by summing up the loss array and dividing it by batches \ No newline at end of file + return np.sum(npc.as_array(self.loss.get_obj()))/batches # return the average loss by summing up the loss array and dividing it by batches From 50909d8587be5752d4ebace03c3ed5549600d238 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Mon, 14 Aug 2023 15:25:57 +0800 Subject: [PATCH 169/337] Update README.md --- README.md | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/README.md b/README.md index 3481cab3..9de1c43c 100644 --- a/README.md +++ b/README.md @@ -204,7 +204,6 @@ nn=n.nn() #create neural network object nn.build() #build the network structure kernel=k.kernel(nn) #create kernel object with the network kernel.process=3 #set the number of processes to train -kernel.data_segment_flag=True #set the flag to segment data for each process kernel.epoch=5 #set the number of epochs to train kernel.batch=32 #set the batch size kernel.PO=1 #use PO1 algorithm for parallel optimization @@ -232,7 +231,6 @@ nn=n.nn() #create neural network object nn.build() #build the network structure kernel=k.kernel(nn) #create kernel object with the network kernel.process=3 #set the number of processes to train -kernel.data_segment_flag=True #set the flag to segment data for each process kernel.epoch=5 #set the number of epochs to train kernel.batch=32 #set the batch size kernel.PO=2 #use PO2 algorithm for parallel optimization @@ -259,7 +257,6 @@ nn=n.nn() #create neural network object nn.build() #build the network structure kernel=k.kernel(nn) #create kernel object with the network kernel.process=3 #set the number of processes to train -kernel.data_segment_flag=True #set the flag to segment data for each process kernel.epoch=5 #set the number of epochs to train kernel.batch=32 #set the batch size kernel.PO=2 #use PO2 algorithm for parallel optimization @@ -287,7 +284,6 @@ nn=n.nn() #create neural network object nn.build() #build the network structure kernel=k.kernel(nn) #create kernel object with the network kernel.process=3 #set the number of processes to train -kernel.data_segment_flag=True #set the flag to segment data for each process kernel.epoch=5 #set the number of epochs to train kernel.batch=32 #set the batch size kernel.PO=3 #use PO3 algorithm for parallel optimization @@ -314,7 +310,6 @@ nn=n.nn() #create neural network object nn.build() #build the network structure kernel=k.kernel(nn) #create kernel object with the network kernel.process=3 #set the number of processes to train -kernel.data_segment_flag=True #set the flag to segment data for each process kernel.epoch=5 #set the number of epochs to train kernel.batch=32 #set the batch size kernel.PO=3 #use PO3 algorithm for parallel optimization @@ -337,7 +332,6 @@ mnist=tf.keras.datasets.mnist #load mnist dataset x_train,x_test =x_train/255.0,x_test/255.0 #normalize data kernel=k.kernel() #create kernel object without a network kernel.process=3 #set the number of processes to train -kernel.data_segment_flag=True #set the flag to segment data for each process kernel.epoch=1 #set the number of epochs to train kernel.batch=32 #set the batch size kernel.PO=3 #use PO3 algorithm for parallel optimization @@ -362,7 +356,6 @@ nn=n.nn() #create neural network object nn.build() #build the network structure kernel=k.kernel(nn) #create kernel object with the network kernel.process=3 #set the number of processes to train -kernel.data_segment_flag=True #set the flag to segment data for each process kernel.epoch=5 #set the number of epochs to train kernel.batch=32 #set the batch size kernel.PO=3 #use PO3 algorithm for parallel optimization @@ -391,7 +384,6 @@ nn.build() #build the network structure kernel=k.kernel(nn) #create kernel object with the network kernel.process=3 #set the number of processes to train kernel.process_t=3 #set the number of processes to test -kernel.data_segment_flag=True #set the flag to segment data for each process kernel.epoch=5 #set the number of epochs to train kernel.batch=32 #set the batch size kernel.PO=3 #use PO3 algorithm for parallel optimization @@ -415,7 +407,6 @@ nn=n.nn() #create neural network object nn.build() #build the network structure kernel=k.kernel(nn) #create kernel object with the network kernel.process=3 #set the number of processes to train -kernel.data_segment_flag=True #set the flag to segment data for each process kernel.epoch=5 #set the number of epochs to train kernel.batch=32 #set the batch size kernel.PO=3 #use PO3 algorithm for parallel optimization @@ -445,7 +436,6 @@ kernel=k.kernel(nn) #create kernel object with the network kernel.stop=True #set the flag to stop training when a condition is met kernel.end_loss=0.7 #set the condition to stop training when the loss is less than 0.7 kernel.process=3 #set the number of processes to train -kernel.data_segment_flag=True #set the flag to segment data for each process kernel.epoch=5 #set the number of epochs to train kernel.batch=32 #set the batch size kernel.PO=3 #use PO3 algorithm for parallel optimization @@ -470,7 +460,6 @@ nn=n.nn() #create neural network object nn.build() #build the network structure kernel=k.kernel(nn) #create kernel object with the network kernel.process=3 #set the number of processes to train -kernel.data_segment_flag=True #set the flag to segment data for each process kernel.epoch=5 #set the number of epochs to train kernel.batch=32 #set the batch size kernel.priority_flag=True #set the flag to use priority scheduling for processes @@ -502,7 +491,6 @@ nn=n.nn() #create neural network object nn.build() #build the network structure kernel=k.kernel(nn) #create kernel object with the network kernel.process=3 #set the number of processes to train -kernel.data_segment_flag=True #set the flag to segment data for each process kernel.epoch=5 #set the number of epochs to train kernel.batch=32 #set the batch size kernel.priority_flag=True #set the flag to use priority scheduling for processes From 752f487845f8535fe8f1afed753774bee7eca016 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Tue, 15 Aug 2023 14:35:03 +0800 Subject: [PATCH 170/337] Update kernel.py --- Note 7.0 documentation/DL/kernel/parallel/kernel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Note 7.0 documentation/DL/kernel/parallel/kernel.py b/Note 7.0 documentation/DL/kernel/parallel/kernel.py index 7fa9a532..52319c71 100644 --- a/Note 7.0 documentation/DL/kernel/parallel/kernel.py +++ b/Note 7.0 documentation/DL/kernel/parallel/kernel.py @@ -17,7 +17,7 @@ def __init__(self,nn=None): self.train_ds=None # the training dataset self.prefetch_batch_size=tf.data.AUTOTUNE # the prefetch batch size for training dataset self.prefetch_batch_size_t=tf.data.AUTOTUNE # the prefetch batch size for testing dataset - self.data_segment_flag=False # a flag to indicate whether to segment the data for each process + self.data_segment_flag=True # a flag to indicate whether to segment the data for each process self.batches=None # the number of batches per epoch self.buffer_size=None # the buffer size for shuffling the data self.priority_flag=False # a flag to indicate whether to use priority optimization From 0b8c9f08b181ca55182b998bad3dc357218c18f1 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Wed, 16 Aug 2023 14:05:25 +0800 Subject: [PATCH 171/337] Delete bert.py --- .../tensorflow/non-parallel/bert.py | 167 ------------------ 1 file changed, 167 deletions(-) delete mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/non-parallel/bert.py diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/non-parallel/bert.py b/Note 7.0 documentation/DL/neural network/tensorflow/non-parallel/bert.py deleted file mode 100644 index 9715477f..00000000 --- a/Note 7.0 documentation/DL/neural network/tensorflow/non-parallel/bert.py +++ /dev/null @@ -1,167 +0,0 @@ -import tensorflow as tf -import tensorflow_hub as hub - - -class bert: - def __init__(self): - text_input=tf.keras.layers.Input(shape=(),dtype=tf.string,name='text') - preprocessing_layer=hub.KerasLayer(tfhub_handle_preprocess,name='preprocessing') - encoder_inputs=preprocessing_layer(text_input) - encoder=hub.KerasLayer(tfhub_handle_encoder,trainable=True,name='BERT_encoder') - outputs=encoder(encoder_inputs) - net=outputs['pooled_output'] - net=tf.keras.layers.Dropout(0.1)(net) - net=tf.keras.layers.Dense(1,activation=None,name='classifier')(net) - self.model=tf.keras.Model(text_input,net) - self.param=self.model.weights - self.opt=tf.keras.optimizers.Adam() - - - def fp(self,data): - return self.model(data) - - - def loss(self,output,labels): - bce=tf.keras.losses.BinaryCrossentropy(from_logits=True) - return bce(output,labels) - - -bert_model_name = 'small_bert/bert_en_uncased_L-4_H-512_A-8' -map_name_to_handle = { - 'bert_en_uncased_L-12_H-768_A-12': - 'https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/3', - 'bert_en_cased_L-12_H-768_A-12': - 'https://tfhub.dev/tensorflow/bert_en_cased_L-12_H-768_A-12/3', - 'bert_multi_cased_L-12_H-768_A-12': - 'https://tfhub.dev/tensorflow/bert_multi_cased_L-12_H-768_A-12/3', - 'small_bert/bert_en_uncased_L-2_H-128_A-2': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-2_H-128_A-2/1', - 'small_bert/bert_en_uncased_L-2_H-256_A-4': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-2_H-256_A-4/1', - 'small_bert/bert_en_uncased_L-2_H-512_A-8': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-2_H-512_A-8/1', - 'small_bert/bert_en_uncased_L-2_H-768_A-12': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-2_H-768_A-12/1', - 'small_bert/bert_en_uncased_L-4_H-128_A-2': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-128_A-2/1', - 'small_bert/bert_en_uncased_L-4_H-256_A-4': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-256_A-4/1', - 'small_bert/bert_en_uncased_L-4_H-512_A-8': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-512_A-8/1', - 'small_bert/bert_en_uncased_L-4_H-768_A-12': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-768_A-12/1', - 'small_bert/bert_en_uncased_L-6_H-128_A-2': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-6_H-128_A-2/1', - 'small_bert/bert_en_uncased_L-6_H-256_A-4': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-6_H-256_A-4/1', - 'small_bert/bert_en_uncased_L-6_H-512_A-8': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-6_H-512_A-8/1', - 'small_bert/bert_en_uncased_L-6_H-768_A-12': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-6_H-768_A-12/1', - 'small_bert/bert_en_uncased_L-8_H-128_A-2': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-8_H-128_A-2/1', - 'small_bert/bert_en_uncased_L-8_H-256_A-4': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-8_H-256_A-4/1', - 'small_bert/bert_en_uncased_L-8_H-512_A-8': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-8_H-512_A-8/1', - 'small_bert/bert_en_uncased_L-8_H-768_A-12': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-8_H-768_A-12/1', - 'small_bert/bert_en_uncased_L-10_H-128_A-2': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-10_H-128_A-2/1', - 'small_bert/bert_en_uncased_L-10_H-256_A-4': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-10_H-256_A-4/1', - 'small_bert/bert_en_uncased_L-10_H-512_A-8': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-10_H-512_A-8/1', - 'small_bert/bert_en_uncased_L-10_H-768_A-12': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-10_H-768_A-12/1', - 'small_bert/bert_en_uncased_L-12_H-128_A-2': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-12_H-128_A-2/1', - 'small_bert/bert_en_uncased_L-12_H-256_A-4': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-12_H-256_A-4/1', - 'small_bert/bert_en_uncased_L-12_H-512_A-8': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-12_H-512_A-8/1', - 'small_bert/bert_en_uncased_L-12_H-768_A-12': - 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-12_H-768_A-12/1', - 'albert_en_base': - 'https://tfhub.dev/tensorflow/albert_en_base/2', - 'electra_small': - 'https://tfhub.dev/google/electra_small/2', - 'electra_base': - 'https://tfhub.dev/google/electra_base/2', - 'experts_pubmed': - 'https://tfhub.dev/google/experts/bert/pubmed/2', - 'experts_wiki_books': - 'https://tfhub.dev/google/experts/bert/wiki_books/2', - 'talking-heads_base': - 'https://tfhub.dev/tensorflow/talkheads_ggelu_bert_en_base/1', -} -map_model_to_preprocess = { - 'bert_en_uncased_L-12_H-768_A-12': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'bert_en_cased_L-12_H-768_A-12': - 'https://tfhub.dev/tensorflow/bert_en_cased_preprocess/3', - 'small_bert/bert_en_uncased_L-2_H-128_A-2': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-2_H-256_A-4': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-2_H-512_A-8': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-2_H-768_A-12': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-4_H-128_A-2': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-4_H-256_A-4': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-4_H-512_A-8': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-4_H-768_A-12': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-6_H-128_A-2': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-6_H-256_A-4': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-6_H-512_A-8': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-6_H-768_A-12': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-8_H-128_A-2': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-8_H-256_A-4': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-8_H-512_A-8': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-8_H-768_A-12': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-10_H-128_A-2': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-10_H-256_A-4': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-10_H-512_A-8': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-10_H-768_A-12': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-12_H-128_A-2': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-12_H-256_A-4': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-12_H-512_A-8': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'small_bert/bert_en_uncased_L-12_H-768_A-12': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'bert_multi_cased_L-12_H-768_A-12': - 'https://tfhub.dev/tensorflow/bert_multi_cased_preprocess/3', - 'albert_en_base': - 'https://tfhub.dev/tensorflow/albert_en_preprocess/3', - 'electra_small': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'electra_base': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'experts_pubmed': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'experts_wiki_books': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', - 'talking-heads_base': - 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', -} -tfhub_handle_encoder=map_name_to_handle[bert_model_name] -tfhub_handle_preprocess=map_model_to_preprocess[bert_model_name] \ No newline at end of file From ed443b0dcb21e20ef666eca26fae308ef4122297 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Wed, 16 Aug 2023 14:06:23 +0800 Subject: [PATCH 172/337] Delete Note 7.0 documentation directory --- Note 7.0 documentation/DL/dl/test.py | 213 ----- Note 7.0 documentation/DL/kernel/kernel.py | 737 ------------------ .../DL/kernel/parallel/kernel.py | 581 -------------- .../DL/kernel/parallel/kernel_pytorch.py | 446 ----------- .../neural network/pytorch/non-parallel/nn.py | 53 -- .../DL/neural network/pytorch/parallel/nn.py | 41 - .../pytorch/parallel/nn_device.py | 45 -- .../neural network/tensorflow/layer/LSTM.py | 37 - .../tensorflow/layer/attn_LSTM.py | 44 -- .../DL/neural network/tensorflow/layer/cnn.py | 50 -- .../tensorflow/layer/cnn_lmix.py | 58 -- .../DL/neural network/tensorflow/layer/nn.py | 27 - .../neural network/tensorflow/layer/nn_acc.py | 32 - .../tensorflow/layer/nn_clipping.py | 40 - .../tensorflow/layer/self_LSTM.py | 45 -- .../tensorflow/layer/transformer.py | 46 -- .../tensorflow/non-parallel/cnn.py | 27 - .../tensorflow/non-parallel/lstm.py | 26 - .../tensorflow/non-parallel/nn.py | 23 - .../tensorflow/non-parallel/nn_acc.py | 28 - .../tensorflow/non-parallel/nn_ol.py | 37 - .../tensorflow/parallel/LSTM.py | 46 -- .../tensorflow/parallel/attn_LSTM.py | 53 -- .../neural network/tensorflow/parallel/cnn.py | 59 -- .../tensorflow/parallel/cnn_lmix.py | 67 -- .../neural network/tensorflow/parallel/nn.py | 39 - .../neural network/tensorflow/parallel/nn_.py | 39 - .../tensorflow/parallel/nn_acc.py | 44 -- .../tensorflow/parallel/nn_attenuate.py | 57 -- .../tensorflow/parallel/nn_clipping.py | 47 -- .../tensorflow/parallel/nn_device.py | 40 - .../tensorflow/parallel/self_LSTM.py | 54 -- .../tensorflow/parallel/transformer.py | 55 -- Note 7.0 documentation/RL/kernel.txt | 4 - Note 7.0 documentation/RL/kernel/kernel.py | 512 ------------ .../RL/kernel/parallel/kernel.py | 664 ---------------- .../RL/kernel/parallel/kernel_pytorch.py | 432 ---------- .../pytorch/non-parallel/DDPG.py | 101 --- .../pytorch/non-parallel/DQN.py | 64 -- .../pytorch/non-parallel/DQN_pr.py | 76 -- .../pytorch/non-parallel/DoubleDQN.py | 65 -- .../pytorch/non-parallel/DuelingDQN.py | 67 -- .../RL/neural network/pytorch/parallel/DQN.py | 65 -- .../tensorflow/non-parallel/DDPG.py | 84 -- .../tensorflow/non-parallel/DQN.py | 46 -- .../tensorflow/non-parallel/DQN_pr.py | 59 -- .../neural network/tensorflow/parallel/DQN.py | 52 -- Note 7.0 documentation/compiler/nc.txt | 33 - Note 7.0 documentation/compiler/nn.n | 26 - Note 7.0 documentation/compiler/nn.py | 25 - 50 files changed, 5611 deletions(-) delete mode 100644 Note 7.0 documentation/DL/dl/test.py delete mode 100644 Note 7.0 documentation/DL/kernel/kernel.py delete mode 100644 Note 7.0 documentation/DL/kernel/parallel/kernel.py delete mode 100644 Note 7.0 documentation/DL/kernel/parallel/kernel_pytorch.py delete mode 100644 Note 7.0 documentation/DL/neural network/pytorch/non-parallel/nn.py delete mode 100644 Note 7.0 documentation/DL/neural network/pytorch/parallel/nn.py delete mode 100644 Note 7.0 documentation/DL/neural network/pytorch/parallel/nn_device.py delete mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/layer/LSTM.py delete mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/layer/attn_LSTM.py delete mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/layer/cnn.py delete mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/layer/cnn_lmix.py delete mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/layer/nn.py delete mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/layer/nn_acc.py delete mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/layer/nn_clipping.py delete mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/layer/self_LSTM.py delete mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/layer/transformer.py delete mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/non-parallel/cnn.py delete mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/non-parallel/lstm.py delete mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/non-parallel/nn.py delete mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/non-parallel/nn_acc.py delete mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/non-parallel/nn_ol.py delete mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/parallel/LSTM.py delete mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/parallel/attn_LSTM.py delete mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/parallel/cnn.py delete mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/parallel/cnn_lmix.py delete mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn.py delete mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn_.py delete mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn_acc.py delete mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn_attenuate.py delete mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn_clipping.py delete mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn_device.py delete mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/parallel/self_LSTM.py delete mode 100644 Note 7.0 documentation/DL/neural network/tensorflow/parallel/transformer.py delete mode 100644 Note 7.0 documentation/RL/kernel.txt delete mode 100644 Note 7.0 documentation/RL/kernel/kernel.py delete mode 100644 Note 7.0 documentation/RL/kernel/parallel/kernel.py delete mode 100644 Note 7.0 documentation/RL/kernel/parallel/kernel_pytorch.py delete mode 100644 Note 7.0 documentation/RL/neural network/pytorch/non-parallel/DDPG.py delete mode 100644 Note 7.0 documentation/RL/neural network/pytorch/non-parallel/DQN.py delete mode 100644 Note 7.0 documentation/RL/neural network/pytorch/non-parallel/DQN_pr.py delete mode 100644 Note 7.0 documentation/RL/neural network/pytorch/non-parallel/DoubleDQN.py delete mode 100644 Note 7.0 documentation/RL/neural network/pytorch/non-parallel/DuelingDQN.py delete mode 100644 Note 7.0 documentation/RL/neural network/pytorch/parallel/DQN.py delete mode 100644 Note 7.0 documentation/RL/neural network/tensorflow/non-parallel/DDPG.py delete mode 100644 Note 7.0 documentation/RL/neural network/tensorflow/non-parallel/DQN.py delete mode 100644 Note 7.0 documentation/RL/neural network/tensorflow/non-parallel/DQN_pr.py delete mode 100644 Note 7.0 documentation/RL/neural network/tensorflow/parallel/DQN.py delete mode 100644 Note 7.0 documentation/compiler/nc.txt delete mode 100644 Note 7.0 documentation/compiler/nn.n delete mode 100644 Note 7.0 documentation/compiler/nn.py diff --git a/Note 7.0 documentation/DL/dl/test.py b/Note 7.0 documentation/DL/dl/test.py deleted file mode 100644 index 13f875a8..00000000 --- a/Note 7.0 documentation/DL/dl/test.py +++ /dev/null @@ -1,213 +0,0 @@ -import tensorflow as tf # import TensorFlow library -import numpy as np # import NumPy library -from multiprocessing import Array # import Array class from multiprocessing module -import numpy.ctypeslib as npc # import ctypeslib module from NumPy - - -@tf.function(jit_compile=True) # use TensorFlow's just-in-time compilation decorator -def test_tf(nn,data,labels): # define a function that tests a TensorFlow network - try: # try to execute the following block of code - try: # try to execute the following block of code - output=nn.fp(data) # get the output of the network by calling its forward propagation method - loss=nn.loss(output,labels) # calculate the loss by calling its loss function - except Exception: # if an exception occurs in the previous block - output,loss=nn.fp(data,labels) # get the output and loss of the network by calling its forward propagation method with both data and labels as arguments - except Exception as e: # if an exception occurs in the previous block - raise e # re-raise the exception - try: # try to execute the following block of code - if hasattr(nn,'accuracy'): # check if the network has an accuracy attribute - acc=nn.accuracy(output,labels) # calculate the accuracy by calling its accuracy function - else: # if the network does not have an accuracy attribute - acc=None # set the accuracy to None - except Exception as e: # if an exception occurs in the previous block - raise e # re-raise the exception - return loss,acc # return the loss and accuracy - - -def test_pytorch(nn,data,labels): # define a function that tests a PyTorch network - try: # try to execute the following block of code - try: # try to execute the following block of code - output=nn.fp(data) # get the output of the network by calling its forward propagation method - loss=nn.loss(output,labels) # calculate the loss by calling its loss function - except Exception: # if an exception occurs in the previous block - output,loss=nn.fp(data,labels) # get the output and loss of the network by calling its forward propagation method with both data and labels as arguments - except Exception as e: # if an exception occurs in the previous block - raise e # re-raise the exception - try: # try to execute the following block of code - if hasattr(nn,'accuracy'): # check if the network has an accuracy attribute - acc=nn.accuracy(output,labels) # calculate the accuracy by calling its accuracy function - else: # if the network does not have an accuracy attribute - acc=None # set the accuracy to None - except Exception as e: # if an exception occurs in the previous block - raise e # re-raise the exception - return loss,acc # return the loss and accuracy - - -def test(nn,test_data,test_labels,platform,batch=None,loss=None,acc_flag='%'): # define a function that tests a generic network on a given platform - if type(nn.param[0])!=list: # check if the first element of nn.param is not a list - test_data=test_data.astype(nn.param[0].dtype.name) # convert test_data to match the data type of nn.param[0] - test_labels=test_labels.astype(nn.param[0].dtype.name) # convert test_labels to match the data type of nn.param[0] - else: # if the first element of nn.param is a list - test_data=test_data.astype(nn.param[0][0].dtype.name) # convert test_data to match the data type of nn.param[0][0] - test_labels=test_labels.astype(nn.param[0][0].dtype.name) # convert test_labels to match the data type of nn.param[0][0] - if batch!=None: # check if batch is not None - total_loss=0 # initialize total_loss to zero - total_acc=0 # initialize total_acc to zero - batches=int((test_data.shape[0]-test_data.shape[0]%batch)/batch) # calculate how many batches are needed - shape0=test_data.shape[0] # store the original shape of test_data along axis 0 - for j in range(batches): # loop through each batch - index1=j*batch # calculate the starting index of data_batch - index2=(j+1)*batch # calculate the ending index of data_batch - data_batch=test_data[index1:index2] # slice test_data to get data_batch - labels_batch=test_labels[index1:index2] # slice test_labels to get labels_batch - if hasattr(platform,'DType'): # check if platform has a DType attribute (indicating TensorFlow) - batch_loss,batch_acc=test_tf(data_batch,labels_batch) # call test_tf function to get batch_loss and batch_acc - else: # if platform does not have a DType attribute (indicating PyTorch) - batch_loss,batch_acc=test_pytorch(data_batch,labels_batch) # call test_pytorch function to get batch_loss and batch_acc - total_loss+=batch_loss # add batch_loss to total_loss - if hasattr(nn,'accuracy'): # check if the network has an accuracy attribute - total_acc+=batch_acc # add batch_acc to total_acc - if shape0%batch!=0: # check if there is a remainder after dividing test_data by batch - batches+=1 # increment batches by one - index1=batches*batch # calculate the starting index of data_batch - index2=batch-(shape0-batches*batch) # calculate the ending index of data_batch - try: # try to execute the following block of code - try: # try to execute the following block of code - data_batch=platform.concat([test_data[index1:],test_data[:index2]],0) # concatenate the remaining test_data and some test_data from the beginning along axis 0 to get data_batch - labels_batch=platform.concat([test_labels[index1:],test_labels[:index2]],0) # concatenate the corresponding labels along axis 0 to get labels_batch - except Exception: # if an exception occurs in the previous block - data_batch=np.concatenate([test_data[index1:],test_data[:index2]],0) # use NumPy's concatenate function instead of platform's concat function - labels_batch=np.concatenate([test_labels[index1:],test_labels[:index2]],0) # use NumPy's concatenate function instead of platform's concat function - except Exception as e: # if an exception occurs in the previous block - raise e # re-raise the exception - if hasattr(platform,'DType'): # check if platform has a DType attribute (indicating TensorFlow) - batch_loss,batch_acc=test_tf(data_batch,labels_batch) # call test_tf function to get batch_loss and batch_acc - else: # if platform does not have a DType attribute (indicating PyTorch) - batch_loss,batch_acc=test_pytorch(data_batch,labels_batch) # call test_pytorch function to get batch_loss and batch_acc - total_loss+=batch_loss # add batch_loss to total_loss - if hasattr(nn,'accuracy'): # check if the network has an accuracy attribute - total_acc+=batch_acc # add batch_acc to total_acc - test_loss=total_loss.numpy()/batches # calculate the average test loss by dividing total_loss by batches and converting it to a NumPy array - test_loss=test_loss.astype(np.float32) # convert test_loss to float32 type - if hasattr(nn,'accuracy'): # check if the network has an accuracy attribute - test_acc=total_acc.numpy()/batches # calculate the average test accuracy by dividing total_acc by batches and converting it to a NumPy array - test_acc=test_acc.astype(np.float32) # convert test_acc to float32 type - else: # if batch is None - if hasattr(platform,'DType'): # check if platform has a DType attribute (indicating TensorFlow) - batch_loss,batch_acc=test_tf(test_data,test_labels) # call test_tf function to get batch_loss and batch_acc - else: # if platform does not have a DType attribute (indicating PyTorch) - batch_loss,batch_acc=test_pytorch(test_data,test_labels) # call test_pytorch function to get batch_loss and batch_acc - test_loss=batch_loss.numpy().astype(np.float32) # convert batch_loss to a NumPy array and float32 type - if hasattr(nn,'accuracy'): # check if the network has an accuracy attribute - test_acc=batch_acc.numpy().astype(np.float32) # convert batch_acc to a NumPy array and float32 type - print('test loss:{0:.6f}'.format(test_loss)) # print the test loss with six decimal places - if hasattr(nn,'accuracy'): # check if the network has an accuracy attribute - if acc_flag=='%': # check if acc_flag is '%' (indicating percentage format) - print('test acc:{0:.1f}'.format(test_acc*100)) # print the test accuracy with one decimal place and multiply by 100 - else: # if acc_flag is not '%' (indicating decimal format) - print('test acc:{0:.6f}'.format(test_acc)) # print the test accuracy with six decimal places - if acc_flag=='%': # check if acc_flag is '%' (indicating percentage format) - return test_loss,test_acc*100 # return the test loss and accuracy multiplied by 100 - else: # if acc_flag is not '%' (indicating decimal format) - return test_loss,test_acc # return the test loss and accuracy - else: # if the network does not have an accuracy attribute - return test_loss # return the test loss - - -class parallel_test: # define a class for parallel testing - def __init__(self,nn,test_data,test_labels,process,batch,prefetch_batch_size=tf.data.AUTOTUNE,test_dataset=None,): # define the constructor method - self.nn=nn # store the neural network as an attribute - if test_data is not None and type(self.nn.param[0])!=list: # check if test_data is not None and the first element of nn.param is not a list - self.test_data=test_data.astype(self.nn.param[0].dtype.name) # convert test_data to match the data type of nn.param[0] - self.test_labels=test_labels.astype(self.nn.param[0].dtype.name) # convert test_labels to match the data type of nn.param[0] - elif test_data is not None: # if test_data is not None but the first element of nn.param is a list - self.test_data=test_data.astype(self.nn.param[0][0].dtype.name) # convert test_data to match the data type of nn.param[0][0] - self.test_labels=test_labels.astype(self.nn.param[0][0].dtype.name) # convert test_labels to match the data type of nn.param[0][0] - self.test_dataset=test_dataset # store the test_dataset as an attribute - self.process=process # store the number of processes as an attribute - self.batch=batch # store the batch size as an attribute - if type(self.nn.param[0])!=list: # check if the first element of nn.param is not a list - self.loss=Array('f',np.zeros([process],dtype=self.nn.param[0].dtype.name)) # create a shared memory array for storing the loss values for each process with the same data type as nn.param[0] - else: # if the first element of nn.param is a list - self.loss=Array('f',np.zeros([process],dtype=self.nn.param[0][0].dtype.name)) # create a shared memory array for storing the loss values for each process with the same data type as nn.param[0][0] - if hasattr(nn,'accuracy'): # check if the network has an accuracy attribute - if type(self.nn.param[0])!=list: # check if the first element of nn.param is not a list - self.acc=Array('f',np.zeros([process],dtype=self.nn.param[0].dtype.name)) # create a shared memory array for storing the accuracy values for each process with the same data type as nn.param[0] - else: # if the first element of nn.param is a list - self.acc=Array('f',np.zeros([process],dtype=self.nn.param[0][0].dtype.name)) # create a shared memory array for storing the accuracy values for each process with the same data type as nn.param[0][0] - self.prefetch_batch_size=prefetch_batch_size # store the prefetch batch size as an attribute - - - def segment_data(self): # define a method that segments the test data and labels into equal parts for each process - if len(self.test_data)!=self.process: # check if the length of test_data is not equal to the number of processes - data=np.split(self.test_data,self.process) # split data into equal parts for each process - labels=np.split(self.test_labels,self.process) # split labels into equal parts for each process - self.test_data=data # assign data to test_data attribute - self.test_labels=labels # assign labels to test_labels attribute - return - - - @tf.function(jit_compile=True) # use TensorFlow's just-in-time compilation decorator - def test_(self,data,labels,p): # define a method that tests a TensorFlow network on a given data, labels, and process index - try: # try to execute the following block of code - try: # try to execute the following block of code - try: # try to execute the following block of code - output=self.nn.fp(data,p) # get the output of the network by calling its forward propagation method with data and process index as arguments - loss=self.nn.loss(output,labels,p) # calculate the loss by calling its loss function with output, labels, and process index as arguments - except Exception: # if an exception occurs in the previous block - output,loss=self.nn.fp(data,labels,p) # get the output and loss of the network by calling its forward propagation method with data, labels, and process index as arguments - except Exception: # if an exception occurs in the previous block - try: # try to execute the following block of code - output=self.nn.fp(data) # get the output of the network by calling its forward propagation method with data as argument - loss=self.nn.loss(output,labels) # calculate the loss by calling its loss function with output and labels as arguments - except Exception: # if an exception occurs in the previous block - output,loss=self.nn.fp(data,labels) # get the output and loss of the network by calling its forward propagation method with data and labels as arguments - except Exception as e: # if an exception occurs in the previous block - raise e # re-raise the exception - try: # try to execute the following block of code - if hasattr(self.nn,'accuracy'): # check if the network has an accuracy attribute - try: # try to execute the following block of code - acc=self.nn.accuracy(output,labels,p) # calculate the accuracy by calling its accuracy function with output, labels, and process index as arguments - except Exception: # if an exception occurs in the previous block - acc=self.nn.accuracy(output,labels) # calculate the accuracy by calling its accuracy function with output and labels as arguments - else: # if the network does not have an accuracy attribute - acc=None # set the accuracy to None - except Exception as e: # if an exception occurs in the previous block - raise e # re-raise the exception - return loss,acc # return the loss and accuracy - - - def test(self,p): # define a method that tests a generic network on a given process index - if self.test_dataset is None: # check if test_dataset is None - test_ds=tf.data.Dataset.from_tensor_slices((self.test_data[p],self.test_labels[p])).batch(self.batch).prefetch(self.prefetch_batch_size) # create a TensorFlow dataset from test_data and test_labels for the given process index, batch it, and prefetch it - elif self.test_dataset is not None and type(self.test_dataset)==list: # check if test_dataset is not None and is a list - test_ds=self.test_dataset[p] # get the test_dataset for the given process index - else: # if test_dataset is not None and is not a list - test_ds=self.test_dataset # use test_dataset as it is - for data_batch,labels_batch in test_ds: # loop through each batch of data and labels in test_ds - try: # try to execute the following block of code - batch_loss,batch_acc=self.test_(data_batch,labels_batch,p) # call test_ method to get batch_loss and batch_acc - except Exception as e: # if an exception occurs in the previous block - raise e # re-raise the exception - if hasattr(self.nn,'accuracy'): # check if the network has an accuracy attribute - self.loss[p]+=batch_loss # add batch_loss to loss array for the given process index - self.acc[p]+=batch_acc # add batch_acc to acc array for the given process index - else: # if the network does not have an accuracy attribute - self.loss[p]+=batch_loss # add batch_loss to loss array for the given process index - return - - - def loss_acc(self): # define a method that calculates and returns the average loss and accuracy across all processes - if self.test_dataset is None: # check if test_dataset is None - shape=len(self.test_data[0])*self.process # calculate the total number of data points by multiplying the length of test_data for one process by the number of processes - elif self.test_dataset is not None and type(self.test_dataset)==list: # check if test_dataset is not None and is a list - shape=len(self.test_dataset[0])*len(self.test_dataset) # calculate the total number of data points by multiplying the length of test_dataset for one process by the length of test_dataset list - else: # if test_dataset is not None and is not a list - shape=len(self.test_dataset)*self.process # calculate the total number of data points by multiplying the length of test_dataset by the number of processes - batches=int((shape-shape%self.batch)/self.batch) # calculate how many batches are needed by dividing shape by batch size and rounding down - if shape%self.batch!=0: # check if there is a remainder after dividing shape by batch size - batches+=1 # increment batches by one - if hasattr(self.nn,'accuracy'): # check if the network has an accuracy attribute - return np.sum(npc.as_array(self.loss.get_obj()))/batches,np.sum(npc.as_array(self.acc.get_obj()))/batches # return the average loss and accuracy by summing up the loss and acc arrays and dividing them by batches - else: # if the network does not have an accuracy attribute - return np.sum(npc.as_array(self.loss.get_obj()))/batches # return the average loss by summing up the loss array and dividing it by batches diff --git a/Note 7.0 documentation/DL/kernel/kernel.py b/Note 7.0 documentation/DL/kernel/kernel.py deleted file mode 100644 index 81c0d91b..00000000 --- a/Note 7.0 documentation/DL/kernel/kernel.py +++ /dev/null @@ -1,737 +0,0 @@ -from tensorflow import function # import the function decorator from tensorflow -from multiprocessing import Process # import the Process class from multiprocessing -import numpy as np # import numpy as np -from Note.DL.dl.test import parallel_test # import the parallel_test class from Note.DL.dl.test -import time # import time - - -class kernel: # define a class - def __init__(self,nn=None): - # define the constructor - self.nn=nn # assign the nn argument to the self.nn attribute - if hasattr(self.nn,'km'): # check if the nn object has the km attribute - self.nn.km=1 # set the km attribute to 1 - self.platform=None # initialize the platform attribute to None - self.batches=None # initialize the batches attribute to None - self.process_t=None # initialize the process_t attribute to None - self.prefetch_batch_size_t=None # initialize the prefetch_batch_size_t attribute to None - self.suspend=False # initialize the suspend attribute to False - self.stop=False # initialize the stop attribute to False - self.stop_flag=False # initialize the stop_flag attribute to False - self.save_epoch=None # initialize the save_epoch attribute to None - self.batch=None # initialize the batch attribute to None - self.epoch=0 # initialize the epoch attribute to 0 - self.end_loss=None # initialize the end_loss attribute to None - self.end_acc=None # initialize the end_acc attribute to None - self.end_test_loss=None # initialize the end_test_loss attribute to None - self.end_test_acc=None # initialize the end_test_acc attribute to None - self.acc_flag='%' # initialize the acc_flag attribute to '%' - self.train_counter=0 # initialize the train_counter attribute to 0 - self.filename='save.dat' # initialize the filename attribute to 'save.dat' - self.train_loss=None # initialize the train_loss attribute to None - self.train_acc=None # initialize the train_acc attribute to None - self.train_loss_list=[] # initialize the train_loss_list attribute to an empty list - self.train_acc_list=[] # initialize the train_acc_list attribute to an empty list - self.test_loss=None # initialize the test_loss attribute to None - self.test_acc=None # initialize the test_acc attribute to None - self.test_loss_list=[] # initialize the test_loss_list attribute to an empty list - self.test_acc_list=[] # initialize the test_acc_list attribute to an empty list - self.test_flag=False # initialize the test_flag attribute to False - self.total_epoch=0 # initialize the total_epoch attribute to 0 - self.time=0 # initialize the time attribute to 0 - self.total_time=0 # initialize the total_time attribute to 0 - - - def data(self,train_data=None,train_labels=None,test_data=None,test_labels=None,train_dataset=None,test_dataset=None): - # define a method for setting up data attributes - if train_data is not None and type(self.nn.param[0])!=list: # check if train_data is not None and the type of the first element of self.nn.param is not list - self.train_data=train_data.astype(self.nn.param[0].dtype.name) # convert the train_data to the same data type as the first element of self.nn.param and assign it to self.train_data - self.train_labels=train_labels.astype(self.nn.param[0].dtype.name) # convert the train_labels to the same data type as the first element of self.nn.param and assign it to self.train_labels - elif train_data is not None: # check if train_data is not None - self.train_data=train_data.astype(self.nn.param[0][0].dtype.name) # convert the train_data to the same data type as the first element of the first element of self.nn.param and assign it to self.train_data - self.train_labels=train_labels.astype(self.nn.param[0][0].dtype.name) # convert the train_labels to the same data type as the first element of the first element of self.nn.param and assign it to self.train_labels - self.train_dataset=train_dataset # assign the train_dataset argument to the self.train_dataset attribute - self.test_data=test_data # assign the test_data argument to the self.test_data attribute - self.test_labels=test_labels # assign the test_labels argument to the self.test_labels attribute - self.test_dataset=test_dataset # assign the test_dataset argument to the self.test_dataset attribute - if test_data is not None or test_dataset is not None: # check if test_data or test_dataset is not None - self.test_flag=True # set the test_flag attribute to True - if train_data is not None: # check if train_data is not None - self.shape0=train_data.shape[0] # get the first dimension of train_data and assign it to self.shape0 - return # return from the method - - - def init(self): - # define a method for initializing some attributes - self.suspend=False # set the suspend attribute to False - self.stop=False # set the stop attribute to False - self.stop_flag=False # set the stop_flag attribute to False - self.save_epoch=None # set the save_epoch attribute to None - self.end_loss=None # set the end_loss attribute to None - self.end_acc=None # set the end_acc attribute to None - self.end_test_loss=None # set the end_test_loss attribute to None - self.end_test_acc=None # set the end_test_acc attribute to None - self.train_loss=None # set the train_loss attribute to None - self.train_acc=None # set the train_acc attribute to None - self.test_loss=None # set the test_loss attribute to None - self.test_acc=None # set the test_acc attribute to None - self.train_loss_list.clear() # clear the train_loss_list attribute - self.train_acc_list.clear() # clear the train_acc_list attribute - self.test_loss_list.clear() # clear the test_loss_list attribute - self.test_acc_list.clear() # clear the test_acc_list attribute - self.test_flag=False # set the test_flag attribute to False - self.train_counter=0 # set the train_counter attribute to 0 - self.epoch=0 # set the epoch attribute to 0 - self.total_epoch=0 # set the total_epoch attribute to 0 - self.time=0 # set the time attribute to 0 - self.total_time=0 # set the total_time attribute to 0 - return # return from the method - - - def end(self): - # define a method for checking if some conditions are met for ending training - if self.end_acc!=None and self.train_acc!=None and self.train_acc>self.end_acc: - return True - elif self.end_loss!=None and self.train_loss!=None and self.train_lossself.end_test_acc: - return True - elif self.end_test_loss!=None and self.test_loss!=None and self.test_lossself.end_acc and self.test_acc>self.end_test_acc: - return True - elif self.end_loss!=None and self.end_test_loss!=None: - if self.train_loss!=None and self.test_loss!=None and self.train_lossself.end_acc: - return True - elif self.end_loss!=None and len(self.train_loss_list)!=0 and self.train_loss_list[-1]self.end_test_acc: - return True - elif self.end_test_loss!=None and len(self.test_loss_list)!=0 and self.test_loss_list[-1]self.end_acc and self.test_acc_list[-1]>self.end_test_acc: - return True - elif self.end_loss!=None and self.end_test_loss!=None: - if len(self.train_loss_list)!=0 and len(self.test_loss_list)!=0 and self.train_loss_list[-1]=self.max_opt: - self.priority_p.value=int(self.priority_p.value) # keep the priority process index as an integer if it reaches the maximum optimization steps - elif self.max_opt==None: - self.priority_p.value=int(self.priority_p.value) # keep the priority process index as an integer if there is no maximum optimization steps - else: - self.priority_p.value=-1 # set the priority process index to -1 if there is no process with maximum optimization steps - if self.priority_flag==True: - self.opt_counter[p]=0 # reset the optimization counter for this process if using priority optimization - if hasattr(self.nn,'attenuate'): - opt_counter=self.nn.opt_counter[0] # get the optimization counter in the neural network model - opt_counter.scatter_update(tf.IndexedSlices(0,p)) # update the optimization counter for this process to 0 - self.nn.opt_counter[0]=opt_counter # set the optimization counter in the neural network model - output,batch_loss,param=self.opt(data_batch,labels_batch,p,lock,g_lock) # perform one optimization step with data and labels batch, process index, lock and global lock as inputs - self.param[7]=param # set the 7th value of the shared dictionary to be the updated model parameters - if self.priority_flag==True: - opt_counter=np.frombuffer(self.opt_counter.get_obj(),dtype='i') # get the optimization counter from the shared array - opt_counter+=1 # increment the optimization counter by 1 for each process - if hasattr(self.nn,'attenuate'): - opt_counter=self.nn.opt_counter[0] # get the optimization counter in the neural network model - opt_counter.assign(opt_counter+1) # increment the optimization counter by 1 for this process - self.nn.opt_counter[0]=opt_counter # set the optimization counter in the neural network model - if hasattr(self.nn,'bc'): - bc=self.nn.bc[0] # get the batch counter in the neural network model - bc.assign_add(1) # increment the batch counter by 1 for this process - self.nn.bc[0]=bc # set the batch counter in the neural network model - try: - if hasattr(self.nn,'accuracy'): - try: - batch_acc=self.nn.accuracy(output,labels_batch,p) # get the accuracy of the output with labels and process index as inputs - except Exception: - batch_acc=self.nn.accuracy(output,labels_batch) # get the accuracy of the output with labels as inputs - except Exception as e: - raise e # raise any exception that occurs - if hasattr(self.nn,'accuracy'): - self.total_loss[p]+=batch_loss # accumulate the total loss for this process - self.total_acc[p]+=batch_acc # accumulate the total accuracy for this process - else: - self.total_loss[p]+=batch_loss # accumulate the total loss for this process - self.batch_counter[p]+=1 # increment the batch counter for this process - if self.PO==1 or self.PO==2: - lock[1].acquire() # acquire the second lock (for printing and saving) - elif lock!=None: - lock.acquire() # acquire a single lock (for printing and saving) - batches=np.sum(self.batch_counter) # get the total number of batches for all processes - if batches>=self.batches: # check whether all batches are finished for this epoch - batch_counter=np.frombuffer(self.batch_counter.get_obj(),dtype='i') # get the batch counter from the shared array - batch_counter*=0 # reset the batch counter to 0 for each process - loss=np.sum(self.total_loss)/batches # calculate the average training loss for this epoch - if hasattr(self.nn,'accuracy'): - train_acc=np.sum(self.total_acc)/batches # calculate the average training accuracy for this epoch - self.total_epoch.value+=1 # increment the total epoch by 1 - self.train_loss.value=loss # set the training loss value to be the average loss - self.train_loss_list.append(loss) # append the average loss to the training loss list - if hasattr(self.nn,'accuracy'): - self.train_acc.value=train_acc # set the training accuracy value to be the average accuracy - self.train_acc_list.append(train_acc) # append the average accuracy to the training accuracy list - if self.test_flag==True: # if using testing data or dataset - if hasattr(self.nn,'accuracy'): - self.test_loss.value,self.test_acc.value=self.test(self.test_data,self.test_labels,test_batch) # get the testing loss and accuracy values with testing data, labels and batch size as inputs - self.test_loss_list.append(self.test_loss.value) # append the testing loss value to the testing loss list - self.test_acc_list.append(self.test_acc.value) # append the testing accuracy value to the testing accuracy list - else: - self.test_loss.value=self.test(self.test_data,self.test_labels,test_batch) # get the testing loss value with testing data, labels and batch size as inputs - self.test_loss_list.append(self.test_loss.value) # append the testing loss value to the testing loss list - self.save_() # call the save_ method to save - self.epoch_counter.value+=1 # increment the epoch counter by 1 - if hasattr(self.nn,'ec'): - ec=self.nn.ec[0] # get the epoch counter in the neural network model - ec.assign_add(1) # increment the epoch counter by 1 for this process - self.nn.ec[0]=ec # set the epoch counter in the neural network model - total_loss=np.frombuffer(self.total_loss.get_obj(),dtype='f') # get the total loss from the shared array - total_loss*=0 # reset the total loss to 0 for each process - if hasattr(self.nn,'accuracy'): - total_acc=np.frombuffer(self.total_acc.get_obj(),dtype='f') # get the total accuracy from the shared array - total_acc*=0 # reset the total accuracy to 0 for each process - if self.PO==1 or self.PO==2: - lock[1].release() # release the second lock (for printing and saving) - elif lock!=None: - lock.release() # release a single lock (for printing and saving) - if self.epoch_counter.value>=self.epoch: - self.param[7]=param # set the 7th value of the shared dictionary to be the updated model parameters - return # return from this method - - - def train(self,p,lock=None,g_lock=None,test_batch=None): - # a method to perform one epoch of training for a given process with different PO strategies and data sources - if self.train_dataset is not None and type(self.train_dataset)==list: - train_ds=self.train_dataset[p] # get the training dataset for this process if it is a list of datasets - elif self.train_dataset is not None: - train_ds=self.train_dataset # get the training dataset if it is a single dataset - else: - if self.data_segment_flag==True: - train_ds=tf.data.Dataset.from_tensor_slices((self.train_data[p],self.train_labels[p])).batch(self.batch).prefetch(self.prefetch_batch_size) # create a tf.data.Dataset object from segmented data and labels for this process with batch size and prefetch size as inputs - elif self.buffer_size!=None: - train_ds=tf.data.Dataset.from_tensor_slices((self.train_data,self.train_labels)).shuffle(self.buffer_size).batch(self.batch).prefetch(self.prefetch_batch_size) # create a tf.data.Dataset object from shuffled data and labels with buffer size, batch size and prefetch size as inputs - else: - train_ds=tf.data.Dataset.from_tensor_slices((self.train_data,self.train_labels)).batch(self.batch).prefetch(self.prefetch_batch_size) # create a tf.data.Dataset object from data and labels with batch size and prefetch size as inputs - self.train7(train_ds,p,test_batch,lock,g_lock) # perform one epoch of training for this process with the training dataset, process index, test batch size, lock and global lock as inputs - return - - - def train_online(self,p,lock=None,g_lock=None): - # a method to perform online training for a given process with different PO strategies - if hasattr(self.nn,'counter'): - self.nn.counter.append(0) # append a 0 to the counter list in the neural network model for this process - while True: - if hasattr(self.nn,'save'): - self.nn.save(self.save,p) # save the model parameters with the save path and process index as inputs if the neural network model has a save method - if hasattr(self.nn,'stop_flag'): - if self.nn.stop_flag==True: # check whether to stop the training by the stop flag in the neural network model - return # return from this method - if hasattr(self.nn,'stop_func'): - if self.nn.stop_func(p): # check whether to stop the training by the stop function in the neural network model with process index as input - return # return from this method - if hasattr(self.nn,'suspend_func'): - self.nn.suspend_func(p) # suspend the training by the suspend function in the neural network model with process index as input - try: - data=self.nn.online(p) # get the online data from the online method in the neural network model with process index as input - except Exception as e: - self.nn.exception_list[p]=e # append any exception that occurs to the exception list in the neural network model for this process - if data=='stop': - return # return from this method if the online data is 'stop' - elif data=='suspend': - self.nn.suspend_func(p) # suspend the training by the suspend function in the neural network model with process index as input if the online data is 'suspend' - try: - if self.PO==2: # if using PO=2 strategy (lock before and after calculating gradients) - if type(g_lock)!=list: - pass # do nothing if g_lock is not a list - elif len(g_lock)==self.process: - ln=p # set ln to be the process index - g_lock=g_lock[ln] # get the global lock for this process - else: - ln=int(np.random.choice(len(g_lock))) # randomly choose a global lock from the list - g_lock=g_lock[ln] # get the global lock for this process - output,loss,param=self.opt(data[0],data[1],p,lock,g_lock) # perform one optimization step with data and labels, process index, lock and global lock as inputs - self.param[7]=param # set the 7th value of the shared dictionary to be the updated model parameters - except Exception as e: - if self.PO==1: - if lock[0].acquire(False): - lock[0].release() # release the first lock if it is acquired by this process - elif self.PO==2: - if g_lock.acquire(False): - g_lock.release() # release the global lock if it is acquired by this process - if lock[0].acquire(False): - lock[0].release() # release the first lock if it is acquired by this process - self.nn.exception_list[p]=e # append any exception that occurs to the exception list in the neural network model for this process - loss=loss.numpy() # convert the loss value to a numpy array - if len(self.nn.train_loss_list)==self.nn.max_length: - del self.nn.train_loss_list[0] # delete the first element of the training loss list in the neural network model if it reaches the maximum length - self.nn.train_loss_list.append(loss) # append the loss value to the training loss list in the neural network model - try: - if hasattr(self.nn,'accuracy'): - try: - acc=self.nn.accuracy(output,data[1]) # get the accuracy of the output with labels as inputs - except Exception: - self.exception_list[p]=True # set the exception flag to True for this process if there is an exception - if len(self.nn.train_acc_list)==self.nn.max_length: - del self.nn.train_acc_list[0] # delete the first element of the training accuracy list in the neural network model if it reaches the maximum length - self.nn.train_acc_list.append(acc) # append the accuracy value to the training accuracy list in the neural network model - except Exception as e: - self.nn.exception_list[p]=e # append any exception that occurs to the exception list in the neural network model for this process - try: - if hasattr(self.nn,'counter'): - count=self.nn.counter[p] # get the counter value for this process in the neural network model - count+=1 # increment the counter value by 1 - self.nn.counter[p]=count # set the counter value for this process in the neural network model - except IndexError: - self.nn.counter.append(0) # append a 0 to the counter list in the neural network model for this process if there is an index error - count=self.nn.counter[p] # get the counter value for this process in the neural network model - count+=1 # increment the counter value by 1 - self.nn.counter[p]=count # set the counter value for this process in the neural network model - return # return from this method - - - @tf.function(jit_compile=True) - def test_(self,data,labels): - # a method to perform one testing step with data and labels as inputs - try: - try: - output=self.nn.fp(data) # get the output of the neural network model with data as input - loss=self.nn.loss(output,labels) # get the loss of the output with labels as input - except Exception: - output,loss=self.nn.fp(data,labels) # get the output and loss of the neural network model with data and labels as inputs - except Exception as e: - raise e # raise any exception that occurs - try: - acc=self.nn.accuracy(output,labels) # get the accuracy of the output with labels as input - except Exception as e: - if hasattr(self.nn,'accuracy'): - raise e # raise any exception that occurs if using accuracy metric - else: - acc=None # set acc to None if not using accuracy metric - return loss,acc # return the loss and accuracy values - - - def test(self,test_data=None,test_labels=None,batch=None): - # a method to perform testing with different data sources and batch sizes - if test_data is not None and type(self.nn.param[0])!=list: - test_data=test_data.astype(self.nn.param[0].dtype.name) # convert the testing data to the same dtype as the model parameters - test_labels=test_labels.astype(self.nn.param[0].dtype.name) # convert the testing labels to the same dtype as the model parameters - elif test_data is not None: - test_data=test_data.astype(self.nn.param[0][0].dtype.name) # convert the testing data to the same dtype as the model parameters - test_labels=test_labels.astype(self.nn.param[0][0].dtype.name) # convert the testing labels to the same dtype as the model parameters - if self.process_t!=None: # if using multiple processes for testing - parallel_test_=parallel_test(self.nn,self.test_data,self.test_labels,self.process_t,batch,self.prefetch_batch_size_t,self.test_dataset) # create a parallel_test object with the neural network model, testing data, labels, number of processes, batch size, prefetch size and dataset as inputs - if type(self.test_data)!=list: - parallel_test_.segment_data() # segment the data for each process if it is not a list of data - for p in range(self.process_t): - Process(target=parallel_test_.test,args=(p,)).start() # start a subprocess to perform testing for each process - try: - if hasattr(self.nn,'accuracy'): - test_loss,test_acc=parallel_test_.loss_acc() # get the testing loss and accuracy values from the parallel_test object - except Exception as e: - if hasattr(self.nn,'accuracy'): - raise e # raise any exception that occurs if using accuracy metric - else: - test_loss=parallel_test_.loss_acc() # get the testing loss value from the parallel_test object - elif batch!=None: # if using a batch size for testing - total_loss=0 # initialize a total loss value - total_acc=0 # initialize a total accuracy value - if self.test_dataset!=None: # if using a testing dataset - batches=0 # initialize a batches counter - for data_batch,labels_batch in self.test_dataset: # iterate over each batch of data and labels in the testing dataset - batches+=1 # increment the batches counter by 1 - batch_loss,batch_acc=self.test_(data_batch,labels_batch) # perform one testing step with data and labels batch as inputs - total_loss+=batch_loss # accumulate the total loss value - if hasattr(self.nn,'accuracy'): - total_acc+=batch_acc # accumulate the total accuracy value if using accuracy metric - test_loss=total_loss.numpy()/batches # calculate the average testing loss value - if hasattr(self.nn,'accuracy'): - test_acc=total_acc.numpy()/batches # calculate the average testing accuracy value if using accuracy metric - else: - shape0=test_data.shape[0] # get the number of samples in testing data - batches=int((shape0-shape0%batch)/batch) # calculate the number of batches for testing data - for j in range(batches): - index1=j*batch # get the start index of each batch - index2=(j+1)*batch # get the end index of each batch - data_batch=test_data[index1:index2] # get a slice of testing data for each batch - labels_batch=test_labels[index1:index2] # get a slice of testing labels for each batch - batch_loss,batch_acc=self.test_(data_batch,labels_batch) # perform one testing step with data and labels batch as inputs - total_loss+=batch_loss # accumulate the total loss value - if hasattr(self.nn,'accuracy'): - total_acc+=batch_acc # accumulate the total accuracy value if using accuracy metric - if shape0%batch!=0: # if there are some remaining samples in testing data - batches+=1 # add one more batch for them - index1=batches*batch # get the start index of the last batch - index2=batch-(shape0-batches*batch) # get the end index of the last batch - data_batch=tf.concat([test_data[index1:],test_data[:index2]],0) # concatenate the remaining samples and some samples from the beginning of testing data to form the last batch - labels_batch=tf.concat([test_labels[index1:],test_labels[:index2]],0) # concatenate the corresponding labels to form the last batch - batch_loss,batch_acc=self.test_(data_batch,labels_batch) # perform one testing step with data and labels batch as inputs - total_loss+=batch_loss # accumulate the total loss value - if hasattr(self.nn,'accuracy'): - total_acc+=batch_acc # accumulate the total accuracy value if using accuracy metric - test_loss=total_loss.numpy()/batches # calculate the average testing loss value - if hasattr(self.nn,'accuracy'): - test_acc=total_acc.numpy()/batches # calculate the average testing accuracy value if using accuracy metric - else: - batch_loss,batch_acc=self.test_(test_data,test_labels) # perform one testing step with testing data and labels as inputs - test_loss=test_loss.numpy() # convert the testing loss value to a numpy array - if hasattr(self.nn,'accuracy'): - test_acc=test_acc.numpy() # convert the testing accuracy value to a numpy array if using accuracy metric - if hasattr(self.nn,'accuracy'): - return test_loss,test_acc # return the testing loss and accuracy values if using accuracy metric - else: - return test_loss # return the testing loss value if not using accuracy metric diff --git a/Note 7.0 documentation/DL/kernel/parallel/kernel_pytorch.py b/Note 7.0 documentation/DL/kernel/parallel/kernel_pytorch.py deleted file mode 100644 index b3263de1..00000000 --- a/Note 7.0 documentation/DL/kernel/parallel/kernel_pytorch.py +++ /dev/null @@ -1,446 +0,0 @@ -import torch # import the torch library -from multiprocessing import Value,Array # import the Value and Array classes from the multiprocessing module -import numpy as np # import the numpy library - - -class kernel: # define a class named kernel - def __init__(self,nn=None): # define the constructor method of the class - self.nn=nn # assign the nn argument to the self.nn attribute - if hasattr(self.nn,'km'): # check if the nn attribute has an attribute named km - self.nn.km=1 # set the km attribute of the nn attribute to 1 - self.process=None # initialize the process attribute to None - self.process_t=None # initialize the process_t attribute to None - self.train_ds=None # initialize the train_ds attribute to None - self.batches_t=None # initialize the batches_t attribute to None - self.shuffle=False # initialize the shuffle attribute to False - self.priority_flag=False # initialize the priority_flag attribute to False - self.max_opt=None # initialize the max_opt attribute to None - self.epoch=None # initialize the epoch attribute to None - self.stop=False # initialize the stop attribute to False - self.save_epoch=None # initialize the save_epoch attribute to None - self.batch=None # initialize the batch attribute to None - self.end_loss=None # initialize the end_loss attribute to None - self.end_acc=None # initialize the end_acc attribute to None - self.end_test_loss=None # initialize the end_test_loss attribute to None - self.end_test_acc=None # initialize the end_test_acc attribute to None - self.acc_flag='%' # initialize the acc_flag attribute to '%' - self.s=None # initialize the s attribute to None - self.saving_one=True # initialize the saving_one attribute to True - self.filename='save.dat' # initialize the filename attribute to 'save.dat' - self.test_flag=False # initialize the test_flag attribute to False - - - def data(self,train_dataset=None,test_dataset=None): - '''define a method named data that takes two arguments: train_dataset and test_dataset''' - - '''This method is used to assign and process the train_dataset and test_dataset attributes, - and also create some arrays for storing some statistics''' - - '''The train_dataset and test_dataset arguments are expected to be PyTorch Dataset objects or lists of PyTorch Dataset objects''' - - '''The train_dataset argument is used for training the neural network''' - - '''The test_dataset argument is used for testing or evaluating the neural network, and it is optional''' - - '''If the test_dataset argument is given, the test_flag attribute will be set to True, - indicating that the test method will be called after each epoch''' - - self.train_dataset=train_dataset # assign the train_dataset argument to the self.train_dataset attribute - self.test_dataset=test_dataset # assign the test_dataset argument to the self.test_dataset attribute - if test_dataset is not None: # check if the test_dataset argument is not None - self.test_flag=True # set the test_flag attribute to True - self.batch_counter=np.zeros(self.process,dtype=np.int32) # create an array of zeros with a length equal to the process attribute and an integer data type, and assign it to the batch_counter attribute - self.total_loss=np.zeros(self.process) # create an array of zeros with a length equal to the process attribute and a float data type, and assign it to the total_loss attribute - if hasattr(self.nn,'accuracy'): # check if the nn attribute has a method named accuracy - if type(self.nn.param[0])!=list: # check if the first element of the param attribute of the nn attribute is not a list - self.total_acc=np.zeros(self.process,dtype=self.nn.param[0].dtype.name) # create an array of zeros with a length equal to the process attribute and a data type equal to the data type of the first element of the param attribute of the nn attribute, and assign it to the total_acc attribute - else: # otherwise - self.total_acc=np.zeros(self.process,dtype=self.nn.param[0][0].dtype.name) # create an array of zeros with a length equal to the process attribute and a data type equal to the data type of the first element of the first element of the param attribute of the nn attribute, and assign it to the total_acc attribute - if self.priority_flag==True: # check if the priority_flag attribute is True - self.opt_counter=np.zeros(self.process,dtype=np.int32) # create an array of zeros with a length equal to the process attribute and an integer data type, and assign it to the opt_counter attribute - return # return nothing - - - def init(self,manager): - '''define a method named init that takes one argument: manager''' - - '''This method is used to initialize some shared variables for multiprocessing, - using the manager argument as a multiprocessing.Manager object''' - - '''The manager argument is expected to be a multiprocessing.Manager object that can create shared variables across processes''' - - self.epoch_counter=Value('i',0) # create a shared variable with an integer data type and an initial value equal to the epoch_counter attribute, and assign it to the epoch_counter attribute - self.batch_counter=Array('i',self.batch_counter) # create a shared array with an integer data type and an initial value equal to the batch_counter attribute, and assign it to the batch_counter attribute - self.total_loss=Array('f',self.total_loss) # create a shared array with a float data type and an initial value equal to the total_loss attribute, and assign it to the total_loss attribute - self.total_epoch=Value('i',0) # create a shared variable with an integer data type and an initial value equal to the total_epoch attribute, and assign it to the total_epoch attribute - self.train_loss=Value('f',0) # create a shared variable with a float data type and an initial value equal to the train_loss attribute, and assign it to the train_loss attribute - self.train_loss_list=manager.list([]) # create a shared list with an initial value equal to the train_loss_list attribute, using the list method of the manager argument, and assign it to the train_loss_list attribute - self.priority_p=Value('i',0) # create a shared variable with an integer data type and an initial value equal to the priority_p attribute, and assign it to the priority_p attribute - if self.test_flag==True: # check if the test_flag attribute is True - self.test_loss=Value('f',0) # create a shared variable with a float data type and an initial value equal to the test_loss attribute, and assign it to the test_loss attribute - self.test_loss_list=manager.list([]) # create a shared list with an initial value equal to the test_loss_list attribute, using the list method of the manager argument, and assign it to the test_loss_list attribute - if hasattr(self.nn,'accuracy'): # check if the nn attribute has a method named accuracy - self.total_acc=Array('f',self.total_acc) # create a shared array with a float data type and an initial value equal to the total_acc attribute, and assign it to the total_acc attribute - self.train_acc=Value('f',0) # create a shared variable with a float data type and an initial value equal to the train_acc attribute, and assign it to the train_acc attribute - self.train_acc_list=manager.list([]) # create a shared list with an initial value equal to the train_acc_list attribute, using the list method of the manager argument, and assign it to the train_acc_list attribute - if self.test_flag==True: # check if the test_flag attribute is True - self.test_acc=Value('f',0) # create a shared variable with a float data type and an initial value equal to the test_acc attribute, and assign it to the test_acc attribute - self.test_acc_list=manager.list([]) # create a shared list with an initial value equal to the test_acc_list attribute, using the list method of the manager argument, and assign it to the test_acc_list attribute - if self.priority_flag==True: # check if the priority_flag attribute is True - self.opt_counter=Array('i',self.opt_counter) # create a shared array with an integer data type and an initial value equal to the opt_counter attribute, and assign it to the opt_counter attribute - try: - self.nn.opt_counter=manager.list([self.nn.opt_counter]) # try to create a shared list with an initial value equal to a list containing the opt_counter attribute of the nn attribute, using the list method of the manager argument, and assign it to the opt_counter attribute of the nn attribute - except Exception: # handle any exception that may occur - self.opt_counter_=manager.list() # create an empty shared list using the list method of the manager argument, and assign it to the opt_counter_ attribute - try: - self.nn.ec=manager.list([self.nn.ec]) # try to create a shared list with an initial value equal to a list containing the ec attribute of the nn attribute, using the list method of the manager argument, and assign it to the ec attribute of the nn attribute - except Exception: # handle any exception that may occur - self.ec_=manager.list() # create an empty shared list using the list method of the manager argument, and assign it to the ec_ attribute - try: - self.nn.bc=manager.list([self.nn.bc]) # try to create a shared list with an initial value equal to a list containing the bc attribute of the nn attribute, using the list method of the manager argument, and assign it to the bc attribute of the nn attribute - except Exception: # handle any exception that may occur - self.bc_=manager.list() # create an empty shared list using the list method of the manager argument, and assign it to the bc_ attribute - self.epoch_=Value('i',0) # create a shared variable with an integer data type and an initial value equal to the epoch_ attribute, and assign it to the epoch_ attribute - self.stop_flag=Value('b',False) # create a shared variable with a boolean data type and an initial value equal to the stop_flag attribute, and assign it to the stop_flag attribute - self.save_flag=Value('b',False) # create a shared variable with a boolean data type and an initial value equal to the save_flag attribute, and assign it to the save_flag attribute - self.file_list=manager.list([]) # create an empty shared list using the list method of the manager argument, and assign it to the file_list attribute - return # return nothing - - - def init_online(self,manager): - '''define a method named init_online that takes one argument: manager''' - - '''This method is used to initialize some shared variables for online training, - using the manager argument as a multiprocessing.Manager object''' - - '''The manager argument is expected to be a multiprocessing.Manager object that can create shared variables across processes''' - - self.nn.train_loss_list=manager.list([]) # create an empty shared list using the list method of the manager argument, and assign it to the train_loss_list attribute of the nn attribute - self.nn.train_acc_list=manager.list([]) # create an empty shared list using the list method of the manager argument, and assign it to the train_acc_list attribute of the nn attribute - self.nn.counter=manager.list([]) # create an empty shared list using the list method of the manager argument, and assign it to the counter attribute of the nn attribute - self.nn.exception_list=manager.list([]) # create an empty shared list using the list method of the manager argument, and assign it to the exception_list attribute of the nn attribute - return # return nothing - - - def end(self): - '''define a method named end that takes no arguments''' - - '''This method is used to check if the training process should be stopped based on some criteria''' - - '''The criteria are based on some attributes of the class, such as end_loss, end_acc, end_test_loss, and end_test_acc''' - - '''These attributes are expected to be numbers that represent some thresholds for training loss, training accuracy, test loss, and test accuracy respectively''' - - '''The method will return a boolean value indicating whether the training process should be stopped or not''' - - if self.end_acc!=None and len(self.train_acc_list)!=0 and self.train_acc_list[-1]>self.end_acc: - return True - elif self.end_loss!=None and len(self.train_loss_list)!=0 and self.train_loss_list[-1]self.end_test_acc: - return True - elif self.end_test_loss!=None and len(self.test_loss_list)!=0 and self.test_loss_list[-1]self.end_acc and self.test_acc_list[-1]>self.end_test_acc: - return True - elif self.end_loss!=None and self.end_test_loss!=None: - if len(self.train_loss_list)!=0 and len(self.test_loss_list)!=0 and self.train_loss_list[-1]=self.max_opt: # check if both max_opt attribute is not None and the element of the opt_counter attribute at the index of the priority_p attribute is greater than or equal to the max_opt attribute - self.priority_p.value=int(self.priority_p.value) # cast the value of the priority_p attribute to an integer - elif self.max_opt==None: # check if max_opt attribute is None - self.priority_p.value=int(self.priority_p.value) # cast the value of the priority_p attribute to an integer - else: # otherwise - self.priority_p.value=-1 # set the value of the priority_p attribute to -1 - if self.priority_flag==True: # check if the priority_flag attribute is True - self.opt_counter[p]=0 # set the element of the opt_counter attribute at the index of p to 0 - if hasattr(self.nn,'attenuate'): # check if the nn attribute has a method named attenuate - opt_counter=self.nn.opt_counter[0] # assign the first element of the opt_counter attribute of the nn attribute to the opt_counter variable - opt_counter[p]=0 # set the element of the opt_counter variable at the index of p to 0 - self.nn.opt_counter[0]=opt_counter # assign the opt_counter variable to the first element of the opt_counter attribute of the nn attribute - output,batch_loss=self.opt(data_batch,labels_batch,p) # call the opt method with the data_batch, labels_batch, and p arguments, and assign the return values to the output and batch_loss variables - if self.priority_flag==True: # check if the priority_flag attribute is True - opt_counter=np.frombuffer(self.opt_counter.get_obj(),dtype='i') # create a numpy array from a buffer object that contains a copy of memory from another object, using an integer data type, and assign it to the opt_counter variable - opt_counter+=1 # increment each element of the opt_counter variable by 1 - if hasattr(self.nn,'attenuate'): # check if the nn attribute has a method named attenuate - opt_counter=self.nn.opt_counter[0] # assign the first element of the opt_counter attribute of the nn attribute to the opt_counter variable - opt_counter+=1 # increment each element of the opt_counter variable by 1 - self.nn.opt_counter[0]=opt_counter # assign the opt_counter variable to the first element of the opt_counter attribute of the nn attribute - if hasattr(self.nn,'bc'): # check if the nn attribute has an attribute named bc - bc=self.nn.bc[0] # assign the first element of the bc attribute of the nn attribute to the bc variable - bc+=1 # increment the bc variable by 1 - self.nn.bc[0]=bc # assign the bc variable to the first element of the bc attribute of the nn attribute - try: # try to execute the following code block - if hasattr(self.nn,'accuracy'): # check if the nn attribute has a method named accuracy - try: # try to execute the following code block - batch_acc=self.nn.accuracy(output,labels_batch,p) # call the accuracy method of the nn attribute with the output, labels_batch, and p arguments, and assign the return value to the batch_acc variable - except Exception: # handle any exception that may occur - batch_acc=self.nn.accuracy(output,labels_batch) # call the accuracy method of the nn attribute with the output and labels_batch arguments, and assign the return value to the batch_acc variable - except Exception as e: # handle any exception that may occur - raise e # raise the exception again - if hasattr(self.nn,'accuracy'): # check if the nn attribute has a method named accuracy - self.total_loss[p]+=batch_loss # increment the element of the total_loss attribute at the index of p by the batch_loss variable - self.total_acc[p]+=batch_acc # increment the element of the total_acc attribute at the index of p by the batch_acc variable - else: # otherwise - self.total_loss[p]+=batch_loss # increment the element of the total_loss attribute at the index of p by the batch_loss variable - self.batch_counter[p]+=1 # increment the element of the batch_counter attribute at the index of p by 1 - if lock is not None: # check if lock argument is not None - lock.acquire() # call the acquire method of lock argument to acquire a lock for synchronization - batches=np.sum(self.batch_counter) # compute the sum of all elements in the batch_counter attribute and assign it to batches variable - if batches>=len(train_loader): # check if batches is greater than or equal to length of train_loader argument - batch_counter=np.frombuffer(self.batch_counter.get_obj(),dtype='i') # create a numpy array from a buffer object that contains a copy of memory from another object, using an integer data type, and assign it to batch_counter variable - batch_counter*=0 # multiply each element of batch_counter variable by 0 - loss=np.sum(self.total_loss)/batches # compute the average of all elements in total_loss attribute and assign it to loss variable - if hasattr(self.nn,'accuracy'): # check if nn attribute has a method named accuracy - train_acc=np.sum(self.total_acc)/batches # compute the average of all elements in total_acc attribute and assign it to train_acc variable - self.total_epoch.value+=1 # increment value of total_epoch attribute by 1 - self.train_loss.value=loss # assign loss variable to value of train_loss attribute - self.train_loss_list.append(loss) # append loss variable to train_loss_list attribute - if hasattr(self.nn,'accuracy'): # check if nn attribute has a method named accuracy - self.train_acc.value=train_acc # assign train_acc variable to value of train_acc attribute - self.train_acc_list.append(train_acc) # append train_acc variable to train_acc_list attribute - if self.test_flag==True: # check if test_flag attribute is True - if hasattr(self.nn,'accuracy'): # check if nn attribute has a method named accuracy - self.test_loss.value,self.test_acc.value=self.test(test_batch) # call the test method with the test_batch argument, and assign the return values to the value of the test_loss attribute and the value of the test_acc attribute - self.test_loss_list.append(self.test_loss.value) # append the value of the test_loss attribute to the test_loss_list attribute - self.test_acc_list.append(self.test_acc.value) # append the value of the test_acc attribute to the test_acc_list attribute - else: # otherwise - self.test_loss.value=self.test(test_batch) # call the test method with the test_batch argument, and assign the return value to the value of the test_loss attribute - self.test_loss_list.append(self.test_loss.value) # append the value of the test_loss attribute to the test_loss_list attribute - self.save_() # call the save_ method to save - self.epoch_counter.value+=1 # increment the value of the epoch_counter attribute by 1 - if hasattr(self.nn,'ec'): # check if the nn attribute has an attribute named ec - ec=self.nn.ec[0] # assign the first element of the ec attribute of the nn attribute to the ec variable - ec+=1 # increment the ec variable by 1 - self.nn.ec[0]=ec # assign the ec variable to the first element of the ec attribute of the nn attribute - total_loss=np.frombuffer(self.total_loss.get_obj(),dtype='f') # create a numpy array from a buffer object that contains a copy of memory from another object, using a float data type, and assign it to total_loss variable - total_loss*=0 # multiply each element of total_loss variable by 0 - if hasattr(self.nn,'accuracy'): # check if nn attribute has a method named accuracy - total_acc=np.frombuffer(self.total_acc.get_obj(),dtype='f') # create a numpy array from a buffer object that contains a copy of memory from another object, using a float data type, and assign it to total_acc variable - total_acc*=0 # multiply each element of total_acc variable by 0 - if lock is not None: # check if lock argument is not None - lock.release() # call the release method of lock argument to release a lock for synchronization - if self.epoch_counter.value>=self.epoch: # check if value of epoch_counter attribute is greater than or equal to epoch attribute - return # return nothing - - - def train(self,p,lock=None,test_batch=None): - '''define a method named train that takes three arguments: p, lock, and test_batch''' - - '''This method is used to train the neural network for one epoch using a given train_dataset, - using the p argument as the index of the process, - using the lock argument as a multiprocessing.Lock object for synchronization, - and using the test_batch argument as a batch of test data for evaluation''' - - '''The p argument is expected to be an integer that represents the index of the process that calls this method''' - - '''The lock argument is expected to be a multiprocessing.Lock object that can be used to acquire and release a lock for synchronization across processes''' - - '''The test_batch argument is expected to be a PyTorch tensor that represents a batch of test data for evaluation''' - if type(self.train_dataset)==list: # check if the train_dataset attribute is a list - train_loader=torch.utils.data.DataLoader(self.train_dataset[p],batch_size=self.batch) # create a PyTorch DataLoader object from the element of the train_dataset attribute at the index of p, using the batch attribute as the batch size, and assign it to the train_loader variable - else: # otherwise - train_loader=torch.utils.data.DataLoader(self.train_dataset,batch_size=self.batch,shuffle=self.shuffle) # create a PyTorch DataLoader object from the train_dataset attribute, using the batch attribute as the batch size and the shuffle attribute as the shuffle flag, and assign it to the train_loader variable - self.train7(train_loader,p,test_batch,lock) # call the train7 method with the train_loader, p, test_batch, and lock arguments - return # return nothing - - - def train_online(self,p,lock=None): - '''define a method named train_online that takes two arguments: p and lock''' - - '''This method is used to train the neural network online using a given online method of the neural network, - using the p argument as the index of the process, - and using the lock argument as a multiprocessing.Lock object for synchronization''' - - '''The p argument is expected to be an integer that represents the index of the process that calls this method''' - - '''The lock argument is expected to be a multiprocessing.Lock object that can be used to acquire and release a lock for synchronization across processes''' - - if hasattr(self.nn,'counter'): # check if the nn attribute has an attribute named counter - self.nn.counter.append(0) # append 0 to the counter attribute of the nn attribute - while True: # enter an infinite loop - if hasattr(self.nn,'save'): # check if the nn attribute has a method named save - self.nn.save(self.save,p) # call the save method of the nn attribute with the save and p arguments - if hasattr(self.nn,'stop_flag'): # check if the nn attribute has an attribute named stop_flag - if self.nn.stop_flag==True: # check if the value of the stop_flag attribute of the nn attribute is True - return # return nothing - if hasattr(self.nn,'stop_func'): # check if the nn attribute has a method named stop_func - if self.nn.stop_func(p): # call the stop_func method of the nn attribute with the p argument, and check if it returns True - return # return nothing - if hasattr(self.nn,'suspend_func'): # check if the nn attribute has a method named suspend_func - self.nn.suspend_func(p) # call the suspend_func method of the nn attribute with the p argument - try: # try to execute the following code block - data=self.nn.online(p) # call the online method of the nn attribute with the p argument, and assign the return value to data variable - except Exception as e: # handle any exception that may occur - self.nn.exception_list[p]=e # assign e to the element of the exception_list attribute of the nn attribute at the index of p - if data=='stop': # check if data is equal to 'stop' - return # return nothing - elif data=='suspend': # check if data is equal to 'suspend' - self.nn.suspend_func(p) # call the suspend_func method of the nn attribute with the p argument - try: # try to execute the following code block - output,loss,param=self.opt(data[0],data[1],p,lock) # call the opt method with the first and second elements of data, p, and lock arguments, and assign the return values to output, loss, and param variables - self.param[7]=param # assign param variable to the seventh element of the param attribute - except Exception as e: # handle any exception that may occur - if lock!=None: # check if lock argument is not None - if lock.acquire(False): # call the acquire method of lock argument with False argument, and check if it returns True - lock.release() # call the release method of lock argument to release a lock for synchronization - self.nn.exception_list[p]=e # assign e to the element of the exception_list attribute of the nn attribute at the index of p - loss=loss.numpy() # convert loss variable to a numpy array and assign it to loss variable - if len(self.nn.train_loss_list)==self.nn.max_length: # check if length of train_loss_list attribute of nn attribute is equal to max_length attribute of nn attribute - del self.nn.train_loss_list[0] # delete the first element of train_loss_list attribute of nn attribute - self.nn.train_loss_list.append(loss) # append loss variable to train_loss_list attribute of nn attribute - try: # try to execute the following code block - if hasattr(self.nn,'accuracy'): # check if nn attribute has a method named accuracy - try: # try to execute the following code block - acc=self.nn.accuracy(output,data[1]) # call the accuracy method of nn attribute with output and second element of data arguments, and assign the return value to acc variable - except Exception: # handle any exception that may occur - self.exception_list[p]=True # assign True to the element of exception_list attribute at the index of p - if len(self.nn.train_acc_list)==self.nn.max_length: # check if length of train_acc_list attribute of nn attribute is equal to max_length attribute of nn attribute - del self.nn.train_acc_list[0] # delete the first element of train_acc_list attribute of nn attribute - self.nn.train_acc_list.append(acc) # append acc variable to train_acc_list attribute of nn attribute - except Exception as e: # handle any exception that may occur - self.nn.exception_list[p]=e # assign e to the element of exception_list attribute of nn attribute at the index of p - try: # try to execute the following code block - if hasattr(self.nn,'counter'): # check if nn attribute has an attribute named counter - count=self.nn.counter[p] # assign the element of counter attribute of nn attribute at the index of p to count variable - count+=1 # increment count variable by 1 - self.nn.counter[p]=count # assign count variable to the element of counter attribute of nn attribute at the index of p - except IndexError: # handle any IndexError that may occur - self.nn.counter.append(0) # append 0 to counter attribute of nn attribute - count=self.nn.counter[p] # assign the element of counter attribute of nn attribute at the index of p to count variable - count+=1 # increment count variable by 1 - self.nn.counter[p]=count # assign count variable to the element of counter attribute of nn attribute at the index of p - return # return nothing - - - def test_(self,data,labels): - '''define a method named test_ that takes two arguments: data and labels''' - - '''This method is used to compute the test loss and test accuracy for a given batch of test data, - using the data and labels arguments as the input and output of the neural network''' - - '''The data and labels arguments are expected to be PyTorch tensors that represent a batch of input and output data for the neural network''' - - try: # try to execute the following code block - try: # try to execute the following code block - output=self.nn.fp(data) # call the fp method of nn attribute with data argument, and assign the return value to output variable - loss=self.nn.loss(output,labels) # call the loss method of nn attribute with output and labels arguments, and assign the return value to loss variable - except Exception: # handle any exception that may occur - output,loss=self.nn.fp(data,labels) # call the fp method of nn attribute with data and labels arguments, and assign the return values to output and loss variables - except Exception as e: # handle any exception that may occur - raise e # raise the exception again - try: # try to execute the following code block - acc=self.nn.accuracy(output,labels) # call the accuracy method of nn attribute with output and labels arguments, and assign the return value to acc variable - except Exception as e: # handle any exception that may occur - if hasattr(self.nn,'accuracy'): # check if nn attribute has a method named accuracy - raise e # raise the exception again - else: # otherwise - acc=None # assign None to acc variable - return loss,acc # return loss and acc variables - - - def test(self,batch=None): - '''define a method named test that takes one argument: batch''' - - '''This method is used to compute the test loss and test accuracy for the whole test dataset or a given batch of test data, - using the batch argument as a batch of test data for evaluation''' - - '''The batch argument is expected to be a PyTorch tensor that represents a batch of test data for evaluation, or None to use the whole test dataset''' - total_loss=0 # initialize total_loss variable to 0 - total_acc=0 # initialize total_acc variable to 0 - batches=0 # initialize batches variable to 0 - if batch is None: # check if batch argument is None - test_loader=torch.utils.data.DataLoader(self.test_dataset,batch_size=self.batch) # create a PyTorch DataLoader object from the test_dataset attribute, using the batch attribute as the batch size, and assign it to the test_loader variable - else: # otherwise - test_loader=[batch] # create a list containing the batch argument and assign it to the test_loader variable - for data_batch,labels_batch in test_loader: # iterate over the batches of test data from the test_loader - batches+=1 # increment batches variable by 1 - batch_loss,batch_acc=self.test_(data_batch,labels_batch) # call the test_ method with the data_batch and labels_batch arguments, and assign the return values to batch_loss and batch_acc variables - total_loss+=batch_loss # increment total_loss variable by batch_loss variable - if hasattr(self.nn,'accuracy'): # check if nn attribute has a method named accuracy - total_acc+=batch_acc # increment total_acc variable by batch_acc variable - test_loss=total_loss.detach().numpy()/batches # compute the average of total_loss variable and convert it to a numpy array and assign it to test_loss variable - if hasattr(self.nn,'accuracy'): # check if nn attribute has a method named accuracy - test_acc=total_acc.detach().numpy()/batches # compute the average of total_acc variable and convert it to a numpy array and assign it to test_acc variable - if hasattr(self.nn,'accuracy'): # check if nn attribute has a method named accuracy - return test_loss,test_acc # return test_loss and test_acc variables - else: # otherwise - return test_loss # return test_loss variable diff --git a/Note 7.0 documentation/DL/neural network/pytorch/non-parallel/nn.py b/Note 7.0 documentation/DL/neural network/pytorch/non-parallel/nn.py deleted file mode 100644 index 1faec0fc..00000000 --- a/Note 7.0 documentation/DL/neural network/pytorch/non-parallel/nn.py +++ /dev/null @@ -1,53 +0,0 @@ -import torch -from torch import nn - - -class NeuralNetwork(nn.Module): - def __init__(self): - super().__init__() - self.flatten=nn.Flatten() - self.linear_relu_stack=nn.Sequential( - nn.Linear(28*28, 512), - nn.ReLU(), - nn.Linear(512, 512), - nn.ReLU(), - nn.Linear(512, 10) - ) - - - def forward(self,x): - x=self.flatten(x) - logits=self.linear_relu_stack(x) - return logits - - -class neuralnetwork: - def __init__(self): - if torch.cuda.is_available(): - self.device=torch.device('cuda') - else: - self.device=torch.device('cpu') - self.model=NeuralNetwork().to(self.device) - self.loss_fn=nn.CrossEntropyLoss() - self.opt=torch.optim.SGD(self.model.parameters(),lr=1e-3) #optimizer,kernel uses it to optimize. - - - def fp(self,x): #forward propagation function,kernel uses it for forward propagation. - pred=self.model(x.to(self.device)) - return pred - - - def loss(self,output,labels): #loss functino,kernel uses it to calculate loss. - loss=self.loss_fn(output,labels.to(self.device)) - return loss - - - def backward(self,loss): #backward function,kernel uses it for backpropagation. - self.optim.zero_grad() - loss.backward() - return - - - def opt(self): #opt function,kernel uses it to optimize. - self.optim.step() - return \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/pytorch/parallel/nn.py b/Note 7.0 documentation/DL/neural network/pytorch/parallel/nn.py deleted file mode 100644 index d9b05567..00000000 --- a/Note 7.0 documentation/DL/neural network/pytorch/parallel/nn.py +++ /dev/null @@ -1,41 +0,0 @@ -import torch # import the PyTorch library -from torch import nn # import the neural network module from PyTorch - -class NeuralNetwork(nn.Module): # define a class for the neural network model - def __init__(self): # define the constructor method - super().__init__() # call the parent class constructor - self.flatten = nn.Flatten() # define a layer to flatten the input - self.linear_relu_stack=nn.Sequential( # define a stack of linear and relu layers - nn.Linear(28*28,512), # a linear layer with 28*28 input features and 512 output features - nn.ReLU(), # a relu activation function - nn.Linear(512,512), # another linear layer with 512 input and output features - nn.ReLU(), # another relu activation function - nn.Linear(512,10) # a final linear layer with 512 input features and 10 output features - ) - - - def forward(self,x): # define the forward method - x = self.flatten(x) # flatten the input x - logits=self.linear_relu_stack(x) # pass x through the linear relu stack and get the logits - return logits # return the logits - - -class neuralnetwork: # define another class for the neural network object - def __init__(self): # define the constructor method - if torch.cuda.is_available(): # check if cuda is available - self.device=torch.device('cuda') # set the device to cuda - else: - self.device=torch.device('cpu') # otherwise set the device to cpu - self.model=NeuralNetwork().to(self.device) # create an instance of the NeuralNetwork class and move it to the device - self.loss_fn=nn.CrossEntropyLoss() # define the loss function as cross entropy loss - self.opt=[torch.optim.SGD(self.model.parameters(),lr=1e-3) for _ in range(7)] # define a list of three stochastic gradient descent optimizers with learning rate 1e-3 - - - def fp(self,x): # define a method for forward propagation - pred=self.model(x.to(self.device)) # get the predictions from the model by passing x to the device - return pred # return the predictions - - - def loss(self,output,labels): # define a method for calculating the loss - loss=self.loss_fn(output,labels.to(self.device)) # compute the loss by passing output and labels to the device - return loss # return the loss \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/pytorch/parallel/nn_device.py b/Note 7.0 documentation/DL/neural network/pytorch/parallel/nn_device.py deleted file mode 100644 index bd0e8a7e..00000000 --- a/Note 7.0 documentation/DL/neural network/pytorch/parallel/nn_device.py +++ /dev/null @@ -1,45 +0,0 @@ -import torch # import the PyTorch library -from torch import nn # import the neural network module from PyTorch -from Note.nn.parallel.assign_device_pytorch import assign_device # import a custom function to assign device for PyTorch tensors - - -class NeuralNetwork(nn.Module): # define a class for the neural network model - def __init__(self): # define the constructor method - super().__init__() # call the parent class constructor - self.flatten = nn.Flatten() # define a layer to flatten the input tensor - self.linear_relu_stack=nn.Sequential( # define a stack of linear and relu layers - nn.Linear(28*28,512), # define a linear layer with input size 28*28 and output size 512 - nn.ReLU(), # define a relu activation function - nn.Linear(512,512), # define another linear layer with input and output size 512 - nn.ReLU(), # define another relu activation function - nn.Linear(512,10) # define the final linear layer with output size 10 - ) - - - def forward(self,x): # define the forward method for the model - x = self.flatten(x) # flatten the input tensor - logits=self.linear_relu_stack(x) # pass the flattened tensor through the linear relu stack - return logits # return the logits as the output - - -class neuralnetwork: # define another class for the neural network object - def __init__(self): # define the constructor method - if torch.cuda.is_available(): # check if cuda device is available - self.device=torch.device('cuda') # set the device to cuda - else: # otherwise - self.device=torch.device('cpu') # set the device to cpu - self.model=NeuralNetwork().to(self.device) # create an instance of the neural network model and move it to the device - self.loss_fn=nn.CrossEntropyLoss() # define the loss function as cross entropy loss - self.opt=[torch.optim.SGD(self.model.parameters(),lr=1e-3) for _ in range(7)] # define a list of optimizers as stochastic gradient descent with learning rate 1e-3 - - - def fp(self,x,p): # define a method for forward propagation - pred=self.model(x.to(assign_device(p,'GPU'))) # assign the device according to the process index p - # get the predictions from the model by passing the input tensor to the device and then to the model - return pred # return the predictions - - - def loss(self,output,labels,p): # define a method for calculating loss - loss=self.loss_fn(output,labels.to(assign_device(p,'GPU'))) # assign the device according to the process index p - # calculate the loss by passing the output and labels tensors to the device and then to the loss function - return loss # return the loss diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/layer/LSTM.py b/Note 7.0 documentation/DL/neural network/tensorflow/layer/LSTM.py deleted file mode 100644 index 272855a3..00000000 --- a/Note 7.0 documentation/DL/neural network/tensorflow/layer/LSTM.py +++ /dev/null @@ -1,37 +0,0 @@ -import tensorflow as tf -from Note.nn.layer.LSTM import LSTM -from Note.nn.layer.dense import dense - -# Define a recurrent neural network class with LSTM layers -class lstm: - def __init__(self): - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.opt=tf.keras.optimizers.Adam() - - - def build(self): - # Create two LSTM layers with 50 hidden units each - self.lstm1=LSTM(weight_shape=(3,50),return_sequence=True) # First LSTM layer that returns the full sequence - self.lstm2=LSTM(weight_shape=(50,50),) # Second LSTM layer that returns the last output only - # Create a dense layer with 10 output units - self.dense=dense([50, 10]) - # Store the parameters of the layers in a list - self.param=[self.lstm1.param, - self.lstm2.param, - self.dense.param, - ] - return - - - def fp(self,data): - # Perform forward propagation on the input data - x=self.lstm1.output(data) # First LSTM layer output - x=self.lstm2.output(x) # Second LSTM layer output - output=self.dense.output(x) # Dense layer output - return output - - - def loss(self, output, labels): - # Compute the loss between the output and the labels - return self.loss_object(labels, output) \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/layer/attn_LSTM.py b/Note 7.0 documentation/DL/neural network/tensorflow/layer/attn_LSTM.py deleted file mode 100644 index 1d41befd..00000000 --- a/Note 7.0 documentation/DL/neural network/tensorflow/layer/attn_LSTM.py +++ /dev/null @@ -1,44 +0,0 @@ -import tensorflow as tf -from Note.nn.layer.LSTM import LSTM -from Note.nn.layer.dense import dense -from Note.nn.layer.attention import attention - -# Define a recurrent neural network class with LSTM layers and attention -class attn_lstm: - def __init__(self): - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.opt=tf.keras.optimizers.Adam() - - - def build(self): - # Create two LSTM layers with 50 hidden units each - self.lstm1=LSTM(weight_shape=(3,50),return_sequence=True) # First LSTM layer that returns the full sequence - self.lstm2=LSTM(weight_shape=(50,50),) # Second LSTM layer that returns the last output only - # Create a dense layer with 10 output units and softmax activation - self.dense=dense([100, 10],activation='softmax') - # Create an attention object with weight shape (50, 50) - self.attn=attention((50,50)) - # Store the parameters of the layers and the attention in a list - self.param=[self.lstm1.param, - self.lstm2.param, - self.dense.param, - self.attn.param] # Add the attention parameters to the list - return - - - def fp(self,data): - # Perform forward propagation on the input data - x=self.lstm1.output(data) # First LSTM layer output, shape (B, N, 50) - y=self.lstm2.output(x) # Second LSTM layer output, shape (B, 50) - # Use the attention object to compute the context vector from x and y, shape (B, 50) - context_vector,_,_=self.attn.output(x,y) - # Concatenate the context vector and the second LSTM output, shape (B, 100) - z=tf.concat([context_vector,y],axis=-1) - output=self.dense.output(z) # Dense layer output with softmax activation, shape (B, 10) - return output - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/layer/cnn.py b/Note 7.0 documentation/DL/neural network/tensorflow/layer/cnn.py deleted file mode 100644 index aa2667f8..00000000 --- a/Note 7.0 documentation/DL/neural network/tensorflow/layer/cnn.py +++ /dev/null @@ -1,50 +0,0 @@ -import tensorflow as tf -from Note.nn.layer.conv2d import conv2d -from Note.nn.layer.dense import dense -from Note.nn.layer.flatten import flatten - -# Define a convolutional neural network class -class cnn: - def __init__(self): - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.opt=tf.keras.optimizers.Adam() - - - def build(self): - # Create three convolutional layers with relu activations - self.conv1=conv2d(weight_shape=(3,3,3,32),activation='relu') - self.conv2=conv2d(weight_shape=(3,3,32,64),activation='relu') - self.conv3=conv2d(weight_shape=(3,3,64,64),activation='relu') - # Create two dense layers with relu and linear activations - self.dense1=dense([64*4*4,64],activation='relu') - self.dense2=dense([64,10]) - # Store the parameters of the layers in a list - self.param = [self.conv1.weight, - self.conv2.weight, - self.conv3.weight, - self.dense1.weight, - self.dense1.bias, - self.dense2.weight, - self.dense2.bias] - return - - - def fp(self,data): - # Perform forward propagation on the input data - x=tf.nn.conv2d(data,self.conv1.weight,strides=(1,1),padding='SAME') # First convolutional layer - x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # First max pooling layer - x=tf.nn.conv2d(x,self.conv2.weight,strides=(1,1),padding='SAME') # Second convolutional layer - x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # Second max pooling layer - x=tf.nn.conv2d(x,self.conv3.weight,strides=(1,1),padding='SAME') # Third convolutional layer - x=flatten(x) # Flatten the output to a vector - x=tf.nn.relu(tf.matmul(x,self.dense1.weight)+self.dense1.bias) # First dense layer with relu activation - x=tf.nn.dropout(x,rate=0.5) # Apply dropout to prevent overfitting - output=tf.matmul(x,self.dense2.weight)+self.dense2.bias # Output layer with linear activation - output=tf.nn.dropout(output,rate=0.5) # Apply dropout to prevent overfitting - return output - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/layer/cnn_lmix.py b/Note 7.0 documentation/DL/neural network/tensorflow/layer/cnn_lmix.py deleted file mode 100644 index 40232919..00000000 --- a/Note 7.0 documentation/DL/neural network/tensorflow/layer/cnn_lmix.py +++ /dev/null @@ -1,58 +0,0 @@ -import tensorflow as tf -from Note.nn.layer.conv2d import conv2d -from Note.nn.layer.dense import dense -from Note.nn.layer.flatten import flatten -from Note.nn.layer.LMix import lmix - -# Define a convolutional neural network class with mixup augmentation -class cnn: - def __init__(self): - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.opt=tf.keras.optimizers.Adam() - # Initialize the alpha parameter for mixup - self.alpha=1.0 - - - def build(self): - # Create three convolutional layers with relu activations - self.conv1=conv2d(weight_shape=(3,3,3,32),activation='relu') - self.conv2=conv2d(weight_shape=(3,3,32,64),activation='relu') - self.conv3=conv2d(weight_shape=(3,3,64,64),activation='relu') - # Create two dense layers with relu and linear activations - self.dense1=dense([64*4*4,64],activation='relu') - self.dense2=dense([64,10]) - # Store the parameters of the layers in a list - self.param = [self.conv1.weight, - self.conv2.weight, - self.conv3.weight, - self.dense1.weight, - self.dense1.bias, - self.dense2.weight, - self.dense2.bias] - return - - - def data_func(self,data_batch,labels_batch): - # Apply mixup augmentation on the input data and labels - return lmix(data_batch,labels_batch,self.alpha) - - - def fp(self,data): - # Perform forward propagation on the input data - x=tf.nn.conv2d(data,self.conv1.weight,strides=(1,1),padding='SAME') # First convolutional layer - x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # First max pooling layer - x=tf.nn.conv2d(x,self.conv2.weight,strides=(1,1),padding='SAME') # Second convolutional layer - x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # Second max pooling layer - x=tf.nn.conv2d(x,self.conv3.weight,strides=(1,1),padding='SAME') # Third convolutional layer - x=flatten(x) # Flatten the output to a vector - x=tf.nn.relu(tf.matmul(x,self.dense1.weight)+self.dense1.bias) # First dense layer with relu activation - x=tf.nn.dropout(x,rate=0.5) # Apply dropout to prevent overfitting - output=tf.matmul(x,self.dense2.weight)+self.dense2.bias # Output layer with linear activation - output=tf.nn.dropout(output,rate=0.5) # Apply dropout to prevent overfitting - return output - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/layer/nn.py b/Note 7.0 documentation/DL/neural network/tensorflow/layer/nn.py deleted file mode 100644 index 77ed7454..00000000 --- a/Note 7.0 documentation/DL/neural network/tensorflow/layer/nn.py +++ /dev/null @@ -1,27 +0,0 @@ -import tensorflow as tf -import Note.nn.layer.dense as d -from Note.nn.layer.flatten import flatten - - -class nn: - def __init__(self): - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.opt=tf.keras.optimizers.Adam() - - - def build(self): - self.layer1=d.dense([784,128],activation='relu') - self.layer2=d.dense([128,10]) - self.param=[self.layer1.param,self.layer2.param] - return - - - def fp(self,data): - data=flatten(data) - output1=self.layer1.output(data) - output2=self.layer2.output(output1) - return output2 - - - def loss(self,output,labels): - return self.loss_object(labels,output) \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/layer/nn_acc.py b/Note 7.0 documentation/DL/neural network/tensorflow/layer/nn_acc.py deleted file mode 100644 index cad20961..00000000 --- a/Note 7.0 documentation/DL/neural network/tensorflow/layer/nn_acc.py +++ /dev/null @@ -1,32 +0,0 @@ -import tensorflow as tf -import Note.nn.layer.dense as d -from Note.nn.layer.flatten import flatten - - -class nn: - def __init__(self): - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.train_accuracy=tf.keras.metrics.SparseCategoricalAccuracy(name='train_accuracy') - self.opt=tf.keras.optimizers.Adam() - - - def build(self): - self.layer1=d.dense([784,128],activation='relu') - self.layer2=d.dense([128,10]) - self.param=[self.layer1.param,self.layer2.param] - return - - - def fp(self,data): - data=flatten(data) - output1=self.layer1.output(data) - output2=self.layer2.output(output1) - return output2 - - - def loss(self,output,labels): - return self.loss_object(labels,output) - - - def accuracy(self,output,labels): - return self.train_accuracy(labels,output) \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/layer/nn_clipping.py b/Note 7.0 documentation/DL/neural network/tensorflow/layer/nn_clipping.py deleted file mode 100644 index b53a43da..00000000 --- a/Note 7.0 documentation/DL/neural network/tensorflow/layer/nn_clipping.py +++ /dev/null @@ -1,40 +0,0 @@ -import tensorflow as tf -import Note.nn.layer.dense as d -from Note.nn.layer.flatten import flatten -from Note.nn.gradient_clipping import gradient_clipping - -# Define a neural network class -class nn: - def __init__(self): - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.opt=tf.keras.optimizers.Adam() - - - def build(self): - # Create two dense layers with relu and linear activations - self.layer1=d.dense([784,128],activation='relu') - self.layer2=d.dense([128,10]) - # Store the parameters of the layers in a list - self.param=[self.layer1.param,self.layer2.param] - return - - - def fp(self,data): - # Perform forward propagation on the input data - data=flatten(data) - output1=self.layer1.output(data) - output2=self.layer2.output(output1) - return output2 - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) - - - def gradient(self,tape,loss): - # Compute the gradients of the loss with respect to the parameters - grads=tape.gradient(loss,self.param) - # Clip the gradients by value to avoid exploding gradients - return gradient_clipping(grads,'value',1.0) \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/layer/self_LSTM.py b/Note 7.0 documentation/DL/neural network/tensorflow/layer/self_LSTM.py deleted file mode 100644 index edaa2420..00000000 --- a/Note 7.0 documentation/DL/neural network/tensorflow/layer/self_LSTM.py +++ /dev/null @@ -1,45 +0,0 @@ -import tensorflow as tf -from Note.nn.layer.LSTM import LSTM -from Note.nn.layer.self_attention import self_attention -from Note.nn.layer.dense import dense -from Note.nn.layer.emdedding import embedding - -# Define a LSTM self attention neural network class -class self_LSTM: - def __init__(self,vocab_size,embed_size,num_heads,num_layers): - self.vocab_size=vocab_size - self.embed_size=embed_size - self.num_heads=num_heads - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.opt=tf.keras.optimizers.Adam() - - - def build(self): - # Create an embedding layer to map input tokens to vectors - self.embedding_layer = embedding(self.vocab_size,self.embed_size) - # Create a LSTM layer to process the input sequence - self.lstm_layer = LSTM(weight_shape=[self.embed_size,self.embed_size],return_sequence=True) - # Create a self attention layer to attend to the LSTM output - self.self_attention_layer = self_attention(weight_shape=[self.embed_size,self.embed_size],num_heads=self.num_heads) - # Create a linear layer to map the attention output to logits - self.logits_layer = dense([self.embed_size,self.vocab_size]) - # Store the parameters of the layers in a list - self.param = [self.embedding_layer.embeddings] - self.param.extend(self.lstm_layer.param) - self.param.extend(self.self_attention_layer.param) - self.param.extend(self.logits_layer.param) - - - def fp(self,data): - # Perform forward propagation on the input data - x = self.embedding_layer.output(data) # shape: (batch_size, seq_len, embed_size) - x = self.lstm_layer.output(x) # shape: (batch_size, seq_len, embed_size) - x, attention_weights = self.self_attention_layer.output(x,self.num_heads) # shape: (batch_size, seq_len, embed_size) - output = self.logits_layer.output(x) # shape: (batch_size, seq_len, vocab_size) - return output - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/layer/transformer.py b/Note 7.0 documentation/DL/neural network/tensorflow/layer/transformer.py deleted file mode 100644 index 47862006..00000000 --- a/Note 7.0 documentation/DL/neural network/tensorflow/layer/transformer.py +++ /dev/null @@ -1,46 +0,0 @@ -import tensorflow as tf -from Note.nn.layer.transformer import transformer -from Note.nn.layer.dense import dense -from Note.nn.layer.emdedding import embedding -from Note.nn.positional_encoding import positional_encoding - -# Define a positional encoding transformer neural network class -class Transformer: - def __init__(self,vocab_size,embed_size,num_heads,num_layers,max_len): - self.vocab_size=vocab_size - self.embed_size=embed_size - self.num_heads=num_heads - self.num_layers=num_layers - self.max_len=max_len - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.opt=tf.keras.optimizers.Adam() - - - def build(self): - # Create an embedding layer to map input tokens to vectors - self.embedding_layer = embedding(self.vocab_size,self.embed_size) - # Create a list of transformer layers - self.transformer_layers = [transformer(weight_shape=[self.embed_size, self.embed_size], num_heads=self.num_heads) for _ in range(self.num_layers)] - # Create a linear layer to map the transformer output to logits - self.logits_layer = dense([self.embed_size,self.vocab_size]) - # Store the parameters of the layers in a list - self.param = [self.embedding_layer.embeddings] - for transformer_layer in self.transformer_layers: - self.param.extend(transformer_layer.param) - self.param.extend(self.logits_layer.param) - - - def fp(self,data): - # Perform forward propagation on the input data - x = self.embedding_layer.output(data) # shape: (batch_size, seq_len, embed_size) - x += positional_encoding(self.max_len,self.embed_size) # shape: (batch_size, seq_len, embed_size) - for transformer_layer in self.transformer_layers: - x = transformer_layer.output(x) # shape: (batch_size, seq_len, embed_size) - output = self.logits_layer.output(x) # shape: (batch_size, seq_len, vocab_size) - return output - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/non-parallel/cnn.py b/Note 7.0 documentation/DL/neural network/tensorflow/non-parallel/cnn.py deleted file mode 100644 index bd7ad1bf..00000000 --- a/Note 7.0 documentation/DL/neural network/tensorflow/non-parallel/cnn.py +++ /dev/null @@ -1,27 +0,0 @@ -import tensorflow as tf -from tensorflow.keras import layers,models - - -class cnn: - def __init__(self): - self.model=models.Sequential() - self.model.add(layers.Conv2D(32,(3,3),activation='relu',input_shape=(32,32,3))) - self.model.add(layers.MaxPooling2D((2,2))) - self.model.add(layers.Conv2D(64,(3,3),activation='relu')) - self.model.add(layers.MaxPooling2D((2,2))) - self.model.add(layers.Conv2D(64,(3,3),activation='relu')) - self.model.add(layers.Flatten()) - self.model.add(layers.Dense(64,activation='relu')) - self.model.add(layers.Dense(10)) - self.param=self.model.weights - self.opt=tf.keras.optimizers.Adam() - - - def fp(self,data): - output=self.model(data) - return output - - - def loss(self,output,labels): - loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - return loss(labels,output) \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/non-parallel/lstm.py b/Note 7.0 documentation/DL/neural network/tensorflow/non-parallel/lstm.py deleted file mode 100644 index 29b9b396..00000000 --- a/Note 7.0 documentation/DL/neural network/tensorflow/non-parallel/lstm.py +++ /dev/null @@ -1,26 +0,0 @@ -import tensorflow as tf - - -class lstm: - def __init__(self,encoder): - self.model=tf.keras.Sequential([ - encoder, - tf.keras.layers.Embedding(len(encoder.get_vocabulary()), 64, mask_zero=True), - tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(64, return_sequences=True)), - tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(32)), - tf.keras.layers.Dense(64, activation='relu'), - tf.keras.layers.Dropout(0.5), - tf.keras.layers.Dense(1) - ]) - self.param=self.model.weights[1:] - self.opt=tf.keras.optimizers.Adam() - - - def fp(self,data): - output=self.model(data) - return output - - - def loss(self,output,labels): - bce=tf.keras.losses.BinaryCrossentropy(from_logits=True) - return bce(labels,output) \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/non-parallel/nn.py b/Note 7.0 documentation/DL/neural network/tensorflow/non-parallel/nn.py deleted file mode 100644 index b4f03a92..00000000 --- a/Note 7.0 documentation/DL/neural network/tensorflow/non-parallel/nn.py +++ /dev/null @@ -1,23 +0,0 @@ -import tensorflow as tf - - -class nn: - def __init__(self): - self.model=tf.keras.models.Sequential([ - tf.keras.layers.Flatten(input_shape=(28, 28)), - tf.keras.layers.Dense(128,activation='relu'), - tf.keras.layers.Dropout(0.2), - tf.keras.layers.Dense(10) - ]) - self.param=self.model.weights #parameter list,kernel uses it list for backpropagation. - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.opt=tf.keras.optimizers.Adam() #optimizer,kernel uses it to optimize. - - - def fp(self,data): #forward propagation function,kernel uses it for forward propagation. - output=self.model(data) - return output - - - def loss(self,output,labels): #loss functino,kernel uses it to calculate loss. - return self.loss_object(labels,output) \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/non-parallel/nn_acc.py b/Note 7.0 documentation/DL/neural network/tensorflow/non-parallel/nn_acc.py deleted file mode 100644 index 0a726603..00000000 --- a/Note 7.0 documentation/DL/neural network/tensorflow/non-parallel/nn_acc.py +++ /dev/null @@ -1,28 +0,0 @@ -import tensorflow as tf - -#An example with accuracy function. -class nn: - def __init__(self): - self.model=tf.keras.models.Sequential([ - tf.keras.layers.Flatten(input_shape=(28, 28)), - tf.keras.layers.Dense(128,activation='relu'), - tf.keras.layers.Dropout(0.2), - tf.keras.layers.Dense(10) - ]) - self.param=self.model.weights #parameter list,kernel uses it list for backpropagation. - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.train_accuracy=tf.keras.metrics.SparseCategoricalAccuracy(name='train_accuracy') - self.opt=tf.keras.optimizers.Adam() #optimizer,kernel uses it to optimize. - - - def fp(self,data): #forward propagation function,kernel uses it for forward propagation. - output=self.model(data) - return output - - - def loss(self,output,labels): #loss functino,kernel uses it to calculate loss. - return self.loss_object(labels,output) - - - def accuracy(self,output,labels): #accuracy function,kernel uses it to calculate accuracy. - return self.train_accuracy(labels,output) \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/non-parallel/nn_ol.py b/Note 7.0 documentation/DL/neural network/tensorflow/non-parallel/nn_ol.py deleted file mode 100644 index 9dfa608d..00000000 --- a/Note 7.0 documentation/DL/neural network/tensorflow/non-parallel/nn_ol.py +++ /dev/null @@ -1,37 +0,0 @@ -import tensorflow as tf # import TensorFlow library -import numpy as np # import NumPy library - -# This is an example of online training. -class nn: # define a class for the neural network - def __init__(self,data,labels): # initialize the network with data and labels - self.train_data=data # store the training data - self.train_labels=labels # store the training labels - self.model=tf.keras.models.Sequential([ # create a sequential model with four layers - tf.keras.layers.Flatten(input_shape=(28, 28)), # flatten the input to a vector of 28*28 elements - tf.keras.layers.Dense(128,activation='relu'), # add a dense layer with 128 units and ReLU activation - tf.keras.layers.Dropout(0.2), # add a dropout layer with 0.2 dropout rate to prevent overfitting - tf.keras.layers.Dense(10) # add a dense layer with 10 units for the output logits - ]) - self.param=self.model.weights # parameter list, kernel uses it list for backpropagation - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) # loss object, kernel uses it to calculate loss. Here we use sparse categorical crossentropy with logits as output - self.opt=tf.keras.optimizers.Adam() # optimizer, kernel uses it to optimize. Here we use Adam optimizer - self.train_loss_list=[] # a list to store the training loss values - self.counter=0 # counter to keep track of the number of online updates - self.max_length=1000 # maximum length of train_loss_list - - - def online(self): # This is simulative online function, kernel uses it to get online data and labels - index=np.random.choice(60000,size=[32]) # randomly sample 32 indices from 0 to 59999 - if self.counter==10000: # if the counter reaches 10000, stop the online training - return 'stop' - else: # otherwise, return the data and labels corresponding to the sampled indices - return [self.train_data[index],self.train_labels[index]] - - - def fp(self,data): # forward propagation function, kernel uses it for forward propagation - output=self.model(data) # pass the data through the model and get the output logits - return output - - - def loss(self,output,labels): # loss function, kernel uses it to calculate loss - return self.loss_object(labels,output) # return the sparse categorical crossentropy loss between labels and output \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/parallel/LSTM.py b/Note 7.0 documentation/DL/neural network/tensorflow/parallel/LSTM.py deleted file mode 100644 index e8686c10..00000000 --- a/Note 7.0 documentation/DL/neural network/tensorflow/parallel/LSTM.py +++ /dev/null @@ -1,46 +0,0 @@ -import tensorflow as tf -from Note.nn.layer.LSTM import LSTM -from Note.nn.layer.dense import dense -from Note.nn.parallel.optimizer import Adam - -# Define a recurrent neural network class with LSTM layers -class lstm: - def __init__(self): - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.optimizer=Adam() - # Initialize a variable to keep track of the batch count - self.bc=tf.Variable(0, dtype=tf.float32) - - - def build(self): - # Create two LSTM layers with 50 hidden units each - self.lstm1=LSTM(weight_shape=(3,50),return_sequence=True) # First LSTM layer that returns the full sequence - self.lstm2=LSTM(weight_shape=(50,50),) # Second LSTM layer that returns the last output only - # Create a dense layer with 10 output units - self.dense=dense([50, 10]) - # Store the parameters of the layers in a list - self.param=[self.lstm1.param, - self.lstm2.param, - self.dense.param, - ] - return - - - def fp(self,data): - # Perform forward propagation on the input data - x=self.lstm1.output(data) # First LSTM layer output - x=self.lstm2.output(x) # Second LSTM layer output - output=self.dense.output(x) # Dense layer output - return output - - - def loss(self, output, labels): - # Compute the loss between the output and the labels - return self.loss_object(labels, output) - - - def opt(self,gradient): - # Perform optimization on the parameters using the gradient and the batch count - param=self.optimizer.opt(gradient,self.param,self.bc[0]) - return param \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/parallel/attn_LSTM.py b/Note 7.0 documentation/DL/neural network/tensorflow/parallel/attn_LSTM.py deleted file mode 100644 index 198e46b1..00000000 --- a/Note 7.0 documentation/DL/neural network/tensorflow/parallel/attn_LSTM.py +++ /dev/null @@ -1,53 +0,0 @@ -import tensorflow as tf -from Note.nn.layer.LSTM import LSTM -from Note.nn.layer.dense import dense -from Note.nn.layer.attention import attention -from Note.nn.parallel.optimizer import Adam - -# Define a recurrent neural network class with LSTM layers and attention -class attn_lstm: - def __init__(self): - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.optimizer=Adam() - # Initialize a variable to keep track of the batch count - self.bc=tf.Variable(0, dtype=tf.float32) - - - def build(self): - # Create two LSTM layers with 50 hidden units each - self.lstm1=LSTM(weight_shape=(3,50),return_sequence=True) # First LSTM layer that returns the full sequence - self.lstm2=LSTM(weight_shape=(50,50),) # Second LSTM layer that returns the last output only - # Create a dense layer with 10 output units and softmax activation - self.dense=dense([100, 10],activation='softmax') - # Create an attention object with weight shape (50, 50) - self.attn=attention((50,50)) - # Store the parameters of the layers and the attention in a list - self.param=[self.lstm1.param, - self.lstm2.param, - self.dense.param, - self.attn.param] # Add the attention parameters to the list - return - - - def fp(self,data): - # Perform forward propagation on the input data - x=self.lstm1.output(data) # First LSTM layer output, shape (B, N, 50) - y=self.lstm2.output(x) # Second LSTM layer output, shape (B, 50) - # Use the attention object to compute the context vector from x and y, shape (B, 50) - context_vector,_,_=self.attn.output(x,y) - # Concatenate the context vector and the second LSTM output, shape (B, 100) - z=tf.concat([context_vector,y],axis=-1) - output=self.dense.output(z) # Dense layer output with softmax activation, shape (B, 10) - return output - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) - - - def opt(self,gradient): - # Perform optimization on the parameters using the gradient and the batch count - param=self.optimizer.opt(gradient,self.param,self.bc[0]) - return param \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/parallel/cnn.py b/Note 7.0 documentation/DL/neural network/tensorflow/parallel/cnn.py deleted file mode 100644 index 59be2e63..00000000 --- a/Note 7.0 documentation/DL/neural network/tensorflow/parallel/cnn.py +++ /dev/null @@ -1,59 +0,0 @@ -import tensorflow as tf -from Note.nn.layer.conv2d import conv2d -from Note.nn.layer.dense import dense -from Note.nn.layer.flatten import flatten -from Note.nn.parallel.optimizer import Adam - -# Define a convolutional neural network class -class cnn: - def __init__(self): - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.optimizer=Adam() - # Initialize a variable to keep track of the batch count - self.bc=tf.Variable(0,dtype=tf.float32) - - - def build(self): - # Create three convolutional layers with relu activations - self.conv1=conv2d(weight_shape=(3,3,3,32),activation='relu') - self.conv2=conv2d(weight_shape=(3,3,32,64),activation='relu') - self.conv3=conv2d(weight_shape=(3,3,64,64),activation='relu') - # Create two dense layers with relu and linear activations - self.dense1=dense([64*4*4,64],activation='relu') - self.dense2=dense([64,10]) - # Store the parameters of the layers in a list - self.param = [self.conv1.weight, - self.conv2.weight, - self.conv3.weight, - self.dense1.weight, - self.dense1.bias, - self.dense2.weight, - self.dense2.bias] - return - - - def fp(self,data): - # Perform forward propagation on the input data - x=tf.nn.conv2d(data,self.conv1.weight,strides=(1,1),padding='SAME') # First convolutional layer - x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # First max pooling layer - x=tf.nn.conv2d(x,self.conv2.weight,strides=(1,1),padding='SAME') # Second convolutional layer - x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # Second max pooling layer - x=tf.nn.conv2d(x,self.conv3.weight,strides=(1,1),padding='SAME') # Third convolutional layer - x=flatten(x) # Flatten the output to a vector - x=tf.nn.relu(tf.matmul(x,self.dense1.weight)+self.dense1.bias) # First dense layer with relu activation - x=tf.nn.dropout(x,rate=0.5) # Apply dropout to prevent overfitting - output=tf.matmul(x,self.dense2.weight)+self.dense2.bias # Output layer with linear activation - output=tf.nn.dropout(output,rate=0.5) # Apply dropout to prevent overfitting - return output - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) - - - def opt(self,gradient): - # Perform optimization on the parameters using the gradient and the batch count - param=self.optimizer.opt(gradient,self.param,self.bc[0]) - return param \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/parallel/cnn_lmix.py b/Note 7.0 documentation/DL/neural network/tensorflow/parallel/cnn_lmix.py deleted file mode 100644 index b6e66458..00000000 --- a/Note 7.0 documentation/DL/neural network/tensorflow/parallel/cnn_lmix.py +++ /dev/null @@ -1,67 +0,0 @@ -import tensorflow as tf -from Note.nn.layer.conv2d import conv2d -from Note.nn.layer.dense import dense -from Note.nn.layer.flatten import flatten -from Note.nn.parallel.optimizer import Adam -from Note.nn.layer.LMix import lmix - -# Define a convolutional neural network class with mixup augmentation -class cnn: - def __init__(self): - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.optimizer=Adam() - # Initialize the alpha parameter for mixup - self.alpha=1.0 - # Initialize a variable to keep track of the batch count - self.bc=tf.Variable(0,dtype=tf.float32) - - - def build(self): - # Create three convolutional layers with relu activations - self.conv1=conv2d(weight_shape=(3,3,3,32),activation='relu') - self.conv2=conv2d(weight_shape=(3,3,32,64),activation='relu') - self.conv3=conv2d(weight_shape=(3,3,64,64),activation='relu') - # Create two dense layers with relu and linear activations - self.dense1=dense([64*4*4,64],activation='relu') - self.dense2=dense([64,10]) - # Store the parameters of the layers in a list - self.param = [self.conv1.weight, - self.conv2.weight, - self.conv3.weight, - self.dense1.weight, - self.dense1.bias, - self.dense2.weight, - self.dense2.bias] - return - - - def data_func(self,data_batch,labels_batch): - # Apply mixup augmentation on the input data and labels - return lmix(data_batch,labels_batch,self.alpha) - - - def fp(self,data): - # Perform forward propagation on the input data - x=tf.nn.conv2d(data,self.conv1.weight,strides=(1,1),padding='SAME') # First convolutional layer - x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # First max pooling layer - x=tf.nn.conv2d(x,self.conv2.weight,strides=(1,1),padding='SAME') # Second convolutional layer - x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # Second max pooling layer - x=tf.nn.conv2d(x,self.conv3.weight,strides=(1,1),padding='SAME') # Third convolutional layer - x=flatten(x) # Flatten the output to a vector - x=tf.nn.relu(tf.matmul(x,self.dense1.weight)+self.dense1.bias) # First dense layer with relu activation - x=tf.nn.dropout(x,rate=0.5) # Apply dropout to prevent overfitting - output=tf.matmul(x,self.dense2.weight)+self.dense2.bias # Output layer with linear activation - output=tf.nn.dropout(output,rate=0.5) # Apply dropout to prevent overfitting - return output - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) - - - def opt(self,gradient): - # Perform optimization on the parameters using the gradient and the batch count - param=self.optimizer.opt(gradient,self.param,self.bc[0]) - return param \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn.py b/Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn.py deleted file mode 100644 index 1b16b5c0..00000000 --- a/Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn.py +++ /dev/null @@ -1,39 +0,0 @@ -import tensorflow as tf -import Note.nn.layer.dense as d -from Note.nn.parallel.optimizer import Momentum -from Note.nn.layer.flatten import flatten - -# Define a neural network class -class nn: - def __init__(self): - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.optimizer=Momentum(0.07,0.7) - - - def build(self): - # Create two dense layers with relu and linear activations - self.layer1=d.dense([784,128],activation='relu') - self.layer2=d.dense([128,10]) - # Store the parameters of the layers in a list - self.param=[self.layer1.param,self.layer2.param] - return - - - def fp(self,data): - # Perform forward propagation on the input data - data=flatten(data) - output1=self.layer1.output(data) - output2=self.layer2.output(output1) - return output2 - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) - - - def opt(self,gradient): - # Perform optimization on the parameters using the gradient - param=self.optimizer.opt(gradient,self.param) - return param \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn_.py b/Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn_.py deleted file mode 100644 index d115cc0c..00000000 --- a/Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn_.py +++ /dev/null @@ -1,39 +0,0 @@ -import tensorflow as tf -from Note.nn.parallel.optimizer import Momentum -from Note.nn.layer.flatten import flatten - -# Define a neural network class -class nn: - def __init__(self): - # Initialize the weights and biases of three layers with random values - self.weight1=tf.Variable(tf.random.normal([784,64],dtype='float64')) - self.bias1=tf.Variable(tf.random.normal([64],dtype='float64')) - self.weight2=tf.Variable(tf.random.normal([64,64],dtype='float64')) - self.bias2=tf.Variable(tf.random.normal([64],dtype='float64')) - self.weight3=tf.Variable(tf.random.normal([64,10],dtype='float64')) - self.bias3=tf.Variable(tf.random.normal([10],dtype='float64')) - # Store the parameters of the layers in a list - self.param=[self.weight1,self.weight2,self.weight3,self.bias1,self.bias2,self.bias3] - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.optimizer=Momentum(0.07,0.7) - - - def fp(self,data): - # Perform forward propagation on the input data - data=flatten(data) - layer1=tf.nn.relu(tf.matmul(data,self.weight1)+self.bias1) # First layer with relu activation - layer2=tf.nn.relu(tf.matmul(layer1,self.weight2)+self.bias2) # Second layer with relu activation - output=tf.matmul(layer2,self.weight3)+self.bias3 # Output layer with linear activation - return output - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) - - - def opt(self,gradient): - # Perform optimization on the parameters using the gradient - param=self.optimizer.opt(gradient,self.param) - return param \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn_acc.py b/Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn_acc.py deleted file mode 100644 index 19b055dc..00000000 --- a/Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn_acc.py +++ /dev/null @@ -1,44 +0,0 @@ -import tensorflow as tf -import Note.nn.layer.dense as d -from Note.nn.parallel.optimizer import Momentum -from Note.nn.layer.flatten import flatten -from Note.nn.accuracy import sparse_categorical_accuracy - -# Define a neural network class -class nn: - def __init__(self): - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.optimizer=Momentum(0.07,0.7) - - - def build(self): - # Create two dense layers with relu and linear activations - self.layer1=d.dense([784,128],activation='relu') - self.layer2=d.dense([128,10]) - # Store the parameters of the layers in a list - self.param=[self.layer1.param,self.layer2.param] - return - - - def fp(self,data): - # Perform forward propagation on the input data - data=flatten(data) - output1=self.layer1.output(data) - output2=self.layer2.output(output1) - return output2 - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) - - - def accuracy(self,output,labels): #accuracy function,kernel uses it to calculate accuracy. - return sparse_categorical_accuracy(labels,output) - - - def opt(self,gradient): - # Perform optimization on the parameters using the gradient - param=self.optimizer.opt(gradient,self.param) - return param \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn_attenuate.py b/Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn_attenuate.py deleted file mode 100644 index d57ee416..00000000 --- a/Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn_attenuate.py +++ /dev/null @@ -1,57 +0,0 @@ -import tensorflow as tf -from tensorflow.python.util import nest -import Note.nn.layer.dense as d -from Note.nn.parallel.optimizer import Momentum -from Note.nn.layer.flatten import flatten -from Note.nn.accuracy import sparse_categorical_accuracy - -# Define a neural network class with gradient attenuation -class nn: - def __init__(self): - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.optimizer=Momentum(0.07,0.7) - # Initialize a variable to keep track of the number of optimization steps - self.opt_counter=tf.Variable(tf.zeros(3,dtype=tf.float32)) - - - def build(self): - # Create two dense layers with relu and linear activations - self.layer1=d.dense([784,128],activation='relu') - self.layer2=d.dense([128,10]) - # Store the parameters of the layers in a list - self.param=[self.layer1.param,self.layer2.param] - return - - - def fp(self,data): - # Perform forward propagation on the input data - data=flatten(data) # Flatten the data to a one-dimensional vector - output1=self.layer1.output(data) # Apply the first layer to the data and get the output - output2=self.layer2.output(output1) # Apply the second layer to the output of the first layer and get the final output - return output2 - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) # Use the loss function to calculate the loss - - - def accuracy(self,output,labels): #accuracy function,kernel uses it to calculate accuracy. - return sparse_categorical_accuracy(labels,output) - - - def attenuate(self,gradient,oc,p): #gradient attenuation function,kernel uses it to calculate attenuation coefficient. - # Apply an exponential decay to the gradient based on the optimization counter - ac=0.9**oc[0][p] #ac:attenuation coefficient - gradient_flat=nest.flatten(gradient) # Flatten the gradient to a one-dimensional vector - for i in range(len(gradient_flat)): #oc:optimization counter - gradient_flat[i]=ac*gradient_flat[i] #p:process number - gradient=nest.pack_sequence_as(gradient,gradient_flat) # Restore the gradient to its original shape - return gradient - - - def opt(self,gradient): - # Perform optimization on the parameters using the gradient - param=self.optimizer.opt(gradient,self.param) # Use the optimizer to update the parameters - return param diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn_clipping.py b/Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn_clipping.py deleted file mode 100644 index a0f757d3..00000000 --- a/Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn_clipping.py +++ /dev/null @@ -1,47 +0,0 @@ -import tensorflow as tf -import Note.nn.layer.dense as d -from Note.nn.parallel.optimizer import Momentum -from Note.nn.layer.flatten import flatten -from Note.nn.gradient_clipping import gradient_clipping - -# Define a neural network class -class nn: - def __init__(self): - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.optimizer=Momentum(0.07,0.7) - - - def build(self): - # Create two dense layers with relu and linear activations - self.layer1=d.dense([784,128],activation='relu') - self.layer2=d.dense([128,10]) - # Store the parameters of the layers in a list - self.param=[self.layer1.param,self.layer2.param] - return - - - def fp(self,data): - # Perform forward propagation on the input data - data=flatten(data) - output1=self.layer1.output(data) - output2=self.layer2.output(output1) - return output2 - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) - - - def gradient(self,tape,loss): - # Compute the gradients of the loss with respect to the parameters - grads=tape.gradient(loss,self.param) - # Clip the gradients by value to avoid exploding gradients - return gradient_clipping(grads,'value',1.0) - - - def opt(self,gradient): - # Perform optimization on the parameters using the gradient - param=self.optimizer.opt(gradient,self.param) - return param \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn_device.py b/Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn_device.py deleted file mode 100644 index c5dd9156..00000000 --- a/Note 7.0 documentation/DL/neural network/tensorflow/parallel/nn_device.py +++ /dev/null @@ -1,40 +0,0 @@ -import tensorflow as tf # import TensorFlow library -import Note.nn.layer.dense as d # import Note's dense layer module -from Note.nn.layer.flatten import flatten # import Note's flatten layer function -from Note.nn.parallel.optimizer import Momentum # import Note's momentum optimizer module -from Note.nn.parallel.assign_device import assign_device # import the function to assign device according to the process index and the device type - - -class nn: # A neural network class example, allocate device for multiple threads - def __init__(self): # initialize the network - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) # loss object, kernel uses it to calculate loss. Here we use sparse categorical crossentropy with logits as output - self.optimizer=Momentum(0.07,0.7) # optimizer, kernel uses it to optimize. Here we use a custom momentum optimizer with learning rate 0.07 and momentum 0.7 - - - def build(self): # build function, kernel uses it to create the network layers - # Create two dense layers with relu and linear activations - self.layer1=d.dense([784,128],activation='relu') # the first layer with 784 input units and 128 output units and ReLU activation - self.layer2=d.dense([128,10]) # the second layer with 128 input units and 10 output units and linear activation - # Store the parameters of the layers in a list - self.param=[self.layer1.param,self.layer2.param] # parameter list of both layers, kernel uses it list for backpropagation - return - - - def fp(self,data,p): # forward propagation function, kernel uses it for forward propagation - with tf.device(assign_device(p,'GPU')): # assign the device according to the process index p - data=flatten(data) # flatten the data to a vector of 784 elements - output1=self.layer1.output(data) # pass the data through the first layer and get the output - output2=self.layer2.output(output1) # pass the output of the first layer through the second layer and get the final output logits - return output2 - - - def loss(self,output,labels,p): # loss function, kernel uses it to calculate loss - with tf.device(assign_device(p,'GPU')): # assign the device according to the process index p - return self.loss_object(labels,output) # return the mean softmax cross entropy loss between labels and output - - - def opt(self,gradient,p): # optimization function, kernel uses it to optimize parameter - # Perform optimization on the parameters using the gradient - with tf.device(assign_device(p,'GPU')): # assign the device according to the process index p - param=self.optimizer.opt(gradient,self.param) # apply the Note's momentum optimizer to update the parameters using the gradient - return param diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/parallel/self_LSTM.py b/Note 7.0 documentation/DL/neural network/tensorflow/parallel/self_LSTM.py deleted file mode 100644 index 85af312a..00000000 --- a/Note 7.0 documentation/DL/neural network/tensorflow/parallel/self_LSTM.py +++ /dev/null @@ -1,54 +0,0 @@ -import tensorflow as tf -from Note.nn.parallel.optimizer import Adam -from Note.nn.layer.LSTM import LSTM -from Note.nn.layer.self_attention import self_attention -from Note.nn.layer.dense import dense -from Note.nn.layer.emdedding import embedding - -# Define a LSTM self attention neural network class -class self_LSTM: - def __init__(self,vocab_size,embed_size,num_heads,num_layers): - self.vocab_size=vocab_size - self.embed_size=embed_size - self.num_heads=num_heads - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.optimizer=Adam() - # Initialize a variable to keep track of the batch count - self.bc=tf.Variable(0,dtype=tf.float32) - - - def build(self): - # Create an embedding layer to map input tokens to vectors - self.embedding_layer = embedding(self.vocab_size,self.embed_size) - # Create a LSTM layer to process the input sequence - self.lstm_layer = LSTM(weight_shape=[self.embed_size,self.embed_size],return_sequence=True) - # Create a self attention layer to attend to the LSTM output - self.self_attention_layer = self_attention(weight_shape=[self.embed_size,self.embed_size],num_heads=self.num_heads) - # Create a linear layer to map the attention output to logits - self.logits_layer = dense([self.embed_size,self.vocab_size]) - # Store the parameters of the layers in a list - self.param = [self.embedding_layer.embeddings] - self.param.extend(self.lstm_layer.param) - self.param.extend(self.self_attention_layer.param) - self.param.extend(self.logits_layer.param) - - - def fp(self,data): - # Perform forward propagation on the input data - x = self.embedding_layer.output(data) # shape: (batch_size, seq_len, embed_size) - x = self.lstm_layer.output(x) # shape: (batch_size, seq_len, embed_size) - x, attention_weights = self.self_attention_layer.output(x,self.num_heads) # shape: (batch_size, seq_len, embed_size) - output = self.logits_layer.output(x) # shape: (batch_size, seq_len, vocab_size) - return output - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) - - - def opt(self,gradient): - # Perform optimization on the parameters using the gradient and the batch count - param=self.optimizer.opt(gradient,self.param,self.bc[0]) - return param \ No newline at end of file diff --git a/Note 7.0 documentation/DL/neural network/tensorflow/parallel/transformer.py b/Note 7.0 documentation/DL/neural network/tensorflow/parallel/transformer.py deleted file mode 100644 index ae052538..00000000 --- a/Note 7.0 documentation/DL/neural network/tensorflow/parallel/transformer.py +++ /dev/null @@ -1,55 +0,0 @@ -import tensorflow as tf -from Note.nn.parallel.optimizer import Adam -from Note.nn.layer.transformer import transformer -from Note.nn.layer.dense import dense -from Note.nn.layer.emdedding import embedding -from Note.nn.positional_encoding import positional_encoding - -# Define a positional encoding transformer neural network class -class Transformer: - def __init__(self,vocab_size,embed_size,num_heads,num_layers,max_len): - self.vocab_size=vocab_size - self.embed_size=embed_size - self.num_heads=num_heads - self.num_layers=num_layers - self.max_len=max_len - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.optimizer=Adam() - # Initialize a variable to keep track of the batch count - self.bc=tf.Variable(0,dtype=tf.float32) - - - def build(self): - # Create an embedding layer to map input tokens to vectors - self.embedding_layer = embedding(self.vocab_size,self.embed_size) - # Create a list of transformer layers - self.transformer_layers = [transformer(weight_shape=[self.embed_size, self.embed_size], num_heads=self.num_heads) for _ in range(self.num_layers)] - # Create a linear layer to map the transformer output to logits - self.logits_layer = dense([self.embed_size,self.vocab_size]) - # Store the parameters of the layers in a list - self.param = [self.embedding_layer.embeddings] - for transformer_layer in self.transformer_layers: - self.param.extend(transformer_layer.param) - self.param.extend(self.logits_layer.param) - - - def fp(self,data): - # Perform forward propagation on the input data - x = self.embedding_layer.output(data) # shape: (batch_size, seq_len, embed_size) - x += positional_encoding(self.max_len,self.embed_size) # shape: (batch_size, seq_len, embed_size) - for transformer_layer in self.transformer_layers: - x = transformer_layer.output(x) # shape: (batch_size, seq_len, embed_size) - output = self.logits_layer.output(x) # shape: (batch_size, seq_len, vocab_size) - return output - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) - - - def opt(self,gradient): - # Perform optimization on the parameters using the gradient and the batch count - param=self.optimizer.opt(gradient,self.param,self.bc[0]) - return param \ No newline at end of file diff --git a/Note 7.0 documentation/RL/kernel.txt b/Note 7.0 documentation/RL/kernel.txt deleted file mode 100644 index 8b6a9b06..00000000 --- a/Note 7.0 documentation/RL/kernel.txt +++ /dev/null @@ -1,4 +0,0 @@ -Pool network: -Pool network use multiprocessing parallel and random add episode in pool,which would make data being uncorrelated in pool, -then pools would be used parallel training agent. -Each process have a pool. diff --git a/Note 7.0 documentation/RL/kernel/kernel.py b/Note 7.0 documentation/RL/kernel/kernel.py deleted file mode 100644 index 72c2c679..00000000 --- a/Note 7.0 documentation/RL/kernel/kernel.py +++ /dev/null @@ -1,512 +0,0 @@ -import tensorflow as tf -import numpy as np -import statistics -import time - - -# This is a class for kernel-based reinforcement learning -class kernel: - # This is the constructor method - def __init__(self,nn=None,save_episode=False): - self.nn=nn # This is the neural network model - if hasattr(self.nn,'km'): # If the model has a kernel matrix attribute - self.nn.km=1 # Set it to 1 - self.platform=None # This is the platform for the model, such as tensorflow or pytorch - self.state_pool=None # This is the pool for storing states - self.action_pool=None # This is the pool for storing actions - self.next_state_pool=None # This is the pool for storing next states - self.reward_pool=None # This is the pool for storing rewards - self.done_pool=None # This is the pool for storing done flags - self.episode_set=[] # This is the list for storing episodes - self.epsilon=None # This is the epsilon value for epsilon-greedy policy - self.episode_step=None # This is the maximum number of steps per episode - self.pool_size=None # This is the maximum size of the pool - self.batch=None # This is the batch size for training - self.update_step=None # This is the frequency of updating the network parameters - self.trial_count=None # This is the number of trials for calculating average reward - self.criterion=None # This is the criterion for stopping training when average reward reaches it - self.reward_list=[] # This is the list for storing rewards per episode - self.suspend=False # This is a flag for suspending training - self.save_epi=None # This is a flag for saving episodes - self.max_episode_count=None # This is the maximum number of episodes to save - self.save_episode=save_episode # This is a flag for saving episodes or not - self.filename='save.dat' # This is the filename for saving data - self.loss=None # This is the loss value for training - self.loss_list=[] # This is the list for storing loss values per episode - self.sc=0 # This is a counter for steps - self.total_episode=0 # This is a counter for episodes - self.time=0 # This is a timer for training time - self.total_time=0 # This is a timer for total time - - - def action_vec(self): # This is a method for creating action probability vector - if self.epsilon!=None: # If epsilon value is not None - self.action_one=np.ones(self.action_count,dtype=np.int8) # Create a vector of ones with length equal to action count - return - - - def init(self): # This is a method for initializing some attributes - try: # Try to execute the following code block - if hasattr(self.nn,'pr'): # If the model has a priority replay attribute - self.nn.pr.TD=np.array(0) # Set it to zero array - except Exception as e: # If an exception occurs - raise e # Raise the exception and exit the method - self.suspend=False # Set the suspend flag to False (not suspending training) - self.save_epi=None # Set the save episode flag to None (not saving episodes) - self.episode_set=[] # Initialize the episode set list to empty list (no episodes saved) - self.state_pool=None # Set the state pool to None (no states stored) - self.action_pool=None # Set the action pool to None (no actions stored) - self.next_state_pool=None # Set the next state pool to None (no next states stored) - self.reward_pool=None # Set the reward pool to None (no rewards stored) - self.done_pool=None # Set the done pool to None (no done flags stored) - self.reward_list=[] # Initialize the reward list to empty list (no rewards recorded) - self.loss=0 # Set the loss value to zero (no loss calculated) - self.loss_list=[] # Initialize the loss list to empty list (no losses recorded) - self.sc=0 # Set the step counter to zero (no steps taken) - self.total_episode=0 # Set the total episode counter to zero (no episodes completed) - self.time=0 # Set the time value to zero (no time elapsed) - self.total_time=0 # Set the total time value to zero (no total time elapsed) - return # Return nothing and exit the method - - - def set_up(self,epsilon=None,episode_step=None,pool_size=None,batch=None,update_step=None,trial_count=None,criterion=None): # This is a method for setting up some parameters - if epsilon!=None: # If epsilon value is given - self.epsilon=epsilon # Set it to the given value - if episode_step!=None: # If episode step value is given - self.episode_step=episode_step # Set it to the given value - if pool_size!=None: # If pool size value is given - self.pool_size=pool_size # Set it to the given value - if batch!=None: # If batch size value is given - self.batch=batch # Set it to the given value - if update_step!=None: # If update step value is given - self.update_step=update_step # Set it to the given value - if trial_count!=None: # If trial count value is given - self.trial_count=trial_count # Set it to the given value - if criterion!=None: # If criterion value is given - self.criterion=criterion # Set it to the given value - self.action_vec() # Call the action vector method - return - - - def epsilon_greedy_policy(self,s): # This is a method for implementing epsilon-greedy policy - action_prob=self.action_one*self.epsilon/len(self.action_one) # Create a vector of action probabilities with epsilon/number of actions for each action - try: - if hasattr(self.platform,'DType'): # If the platform is tensorflow - best_a=np.argmax(self.nn.nn.fp(s)) # Get the best action by using the network's forward propagation method on the state - action_prob[best_a]+=1-self.epsilon # Add 1-epsilon to the best action's probability - else: # If the platform is pytorch - s=self.platform.tensor(s,dtype=self.platform.float).to(self.nn.device) # Convert the state to a tensor with float type and send it to the device (cpu or gpu) - best_a=self.nn.nn(s).argmax() # Get the best action by using the network's forward method on the state - action_prob[best_a.numpy()]+=1-self.epsilon # Add 1-epsilon to the best action's probability and convert it to numpy array - except Exception as e: - raise e - return action_prob # Return the action probability vector - - - def opt(self,state_batch,action_batch,next_state_batch,reward_batch,done_batch): # This is a method for optimizing the network parameters with a batch of data - if hasattr(self.platform,'DType'): # If the platform is tensorflow - loss=self.tf_opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch) # Call the tensorflow optimization method with the data batch and get the loss value - else: # If the platform is pytorch - loss=self.pytorch_opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch) # Call the pytorch optimization method with the data batch and get the loss value - return loss # Return the loss value - - - def opt_ol(self,state,action,next_state,reward,done): # This is a method for optimizing the network parameters with online data (one sample) - if hasattr(self.platform,'DType'): # If the platform is tensorflow - loss=self.tf_opt(state,action,next_state,reward,done) # Call the tensorflow optimization method with the online data and get the loss value - else: # If the platform is pytorch - loss=self.pytorch_opt(state,action,next_state,reward,done) # Call the pytorch optimization method with the online data and get the loss value - return loss # Return the loss value - - - def pool(self,s,a,next_s,r,done): # This is a method for storing data in the pool - if type(self.state_pool)!=np.ndarray and self.state_pool==None: # If this is the first time to store data in the pool (the pool is None) - self.state_pool=s # Set the state pool to be equal to s (the state) - if type(a)==int: # If a (the action) is an integer (discrete action space) - a=np.array(a) # Convert it to a numpy array - self.action_pool=np.expand_dims(a,axis=0) # Expand its dimension by one and set it as the action pool - else: # If a (the action) is not an integer (continuous action space) - self.action_pool=a # Set it as the action pool - self.next_state_pool=np.expand_dims(next_s,axis=0) # Expand the dimension of next_s (the next state) by one and set it as the next state pool - self.reward_pool=np.expand_dims(r,axis=0) # Expand the dimension of r (the reward) by one and set it as the reward pool - self.done_pool=np.expand_dims(done,axis=0) # Expand the dimension of done (the done flag) by one and set it as the done pool - else: # If this is not the first time to store data in the pool (the pool is not None) - self.state_pool=np.concatenate((self.state_pool,s),0) # Concatenate s (the state) with the state pool along the first axis - if type(a)==int: # If a (the action) is an integer (discrete action space) - a=np.array(a) # Convert it to a numpy array - self.action_pool=np.concatenate((self.action_pool,np.expand_dims(a,axis=0)),0) # Expand its dimension by one and concatenate it with the action pool along the first axis - else: # If a (the action) is not an integer (continuous action space) - self.action_pool=np.concatenate((self.action_pool,a),0) # Concatenate a (the action) with the action pool along the first axis - self.next_state_pool=np.concatenate((self.next_state_pool,np.expand_dims(next_s,axis=0)),0) # Expand the dimension of next_s (the next state) by one and concatenate it with the next state pool along the first axis - self.reward_pool=np.concatenate((self.reward_pool,np.expand_dims(r,axis=0)),0) # Expand the dimension of r (the reward) by one and concatenate it with the reward pool along the first axis - self.done_pool=np.concatenate((self.done_pool,np.expand_dims(done,axis=0)),0) # Expand the dimension of done (the done flag) by one and concatenate it with the done pool along the first axis - if len(self.state_pool)>self.pool_size: # If the length of the state pool exceeds the maximum size of the pool - self.state_pool=self.state_pool[1:] # Remove the first element of the state pool - self.action_pool=self.action_pool[1:] # Remove the first element of the action pool - self.next_state_pool=self.next_state_pool[1:] # Remove the first element of the next state pool - self.reward_pool=self.reward_pool[1:] # Remove the first element of the reward pool - self.done_pool=self.done_pool[1:] # Remove the first element of the done pool - return # Return nothing - - - def choose_action(self,s): # This is a method for choosing an action based on a state - if hasattr(self.nn,'nn'): # If the model has a network attribute - if hasattr(self.platform,'DType'): # If the platform is tensorflow - if self.epsilon==None: # If epsilon value is None - self.epsilon=self.nn.epsilon(self.sc) # Set it to be equal to the model's epsilon method with sc (step counter) as input - action_prob=self.epsilon_greedy_policy(s) # Get the action probability vector by using epsilon-greedy policy method with s (state) as input - a=np.random.choice(self.action_count,p=action_prob) # Choose an action randomly according to the action probability vector - else: # If the platform is pytorch - if self.epsilon==None: # If epsilon value is None - self.epsilon=self.nn.epsilon(self.sc) # Set it to be equal to the model's epsilon method with sc (step counter) as input - action_prob=self.epsilon_greedy_policy(s) # Get the action probability vector by using epsilon-greedy policy method with s (state) as input - a=np.random.choice(self.action_count,p=action_prob.numpy()) # Choose an action randomly according to the action probability vector converted to numpy array - else: # If the model does not have a network attribute - if hasattr(self.nn,'action'): # If the model has an action attribute - if hasattr(self.platform,'DType'): # If the platform is tensorflow - if self.epsilon==None: # If epsilon value is None - self.epsilon=self.nn.epsilon(self.sc) # Set it to be equal to the model's epsilon method with sc (step counter) as input - if hasattr(self.nn,'discriminator'): # If the model has a discriminator attribute - a=self.nn.action(s) # Get an action by using model's action method with s (state) as input - reward=self.nn.discriminator(s,a) # Get a reward by using model's discriminator method with s (state) and a (action) as inputs - s=np.squeeze(s) # Squeeze s (state) to remove redundant - else: # If the model does not have a discriminator attribute - a=self.nn.action(s).numpy() # Get an action by using model's action method with s (state) as input and convert it to numpy array - else: # If the platform is pytorch - if self.epsilon==None: # If epsilon value is None - self.epsilon=self.nn.epsilon(self.sc) # Set it to be equal to the model's epsilon method with sc (step counter) as input - if hasattr(self.nn,'discriminator'): # If the model has a discriminator attribute - a=self.nn.action(s) # Get an action by using model's action method with s (state) as input - reward=self.nn.discriminator(s,a) # Get a reward by using model's discriminator method with s (state) and a (action) as inputs - s=np.squeeze(s) # Squeeze s (state) to remove redundant dimensions - else: # If the model does not have a discriminator attribute - a=self.nn.action(s).detach().numpy() # Get an action by using model's action method with s (state) as input and convert it to numpy array - else: # If the model does not have an action attribute - if hasattr(self.platform,'DType'): # If the platform is tensorflow - a=(self.nn.actor.fp(s)+self.nn.noise()).numpy() # Get an action by using model's actor network's forward propagation method with s (state) and noise as inputs and convert it to numpy array - else: # If the platform is pytorch - s=self.platform.tensor(s,dtype=self.platform.float).to(self.nn.device) # Convert the state to a tensor with float type and send it to the device (cpu or gpu) - a=(self.nn.actor(s)+self.nn.noise()).detach().numpy() # Get an action by using model's actor network's forward method with s (state) and noise as inputs and convert it to numpy array - if hasattr(self.nn,'discriminator'): # If the model has a discriminator attribute - return a,reward # Return the action and the reward - else: # If the model does not have a discriminator attribute - return a,None # Return the action and None - - - def _train(self): # This is a method for training the network with data from the pool - if len(self.state_pool)self.pool_size: # If the length of the state pool exceeds the maximum size of the pool - TD=np.array(0) # Create a zero array for TD value - self.nn.pr.TD=np.append(TD,self.nn.pr.TD[2:]) # Append TD value to the TD array of priority replay and remove the first two elements - self.reward=r+self.reward # Add r (reward) to total reward value - loss=self._train() # Call the _train method and get loss value - self.sc+=1 # Increase sc (step counter) by one - if done: # If done flag is True - if self.save_episode==True: # If save episode flag is True - episode=[s,a,next_s,r] # Set episode list to be equal to data - self.reward_list.append(self.reward) # Append total reward value to reward list - return loss,episode,done # Return loss value, episode list, and done flag - elif self.save_episode==True: # If save episode flag is True and done flag is False - episode=[s,a,next_s,r] # Set episode list to be equal to data - s=next_s # Set s (state) to be equal to next_s (next state) - else: # If episode step value is not None (there is a limit on steps per episode) - for _ in range(self.episode_step): # For each step in episode step value - s=np.expand_dims(s,axis=0) # Expand s (state) dimension by one - a,reward=self.choose_action(s) # Get an action and a reward by using choose action method with s (state) as input - next_s,r,done=self.nn.env(a) # Get next state, reward, and done flag by using model's environment method with a (action) as input - if hasattr(self.platform,'DType'): # If the platform is tensorflow - if type(self.nn.param[0])!=list: # If the first element of model's parameter list is not a list (single network) - next_s=np.array(next_s,self.nn.param[0].dtype.name) # Convert next_s (next state) to a numpy array with the same data type as the first element of model's parameter list - r=np.array(r,self.nn.param[0].dtype.name) # Convert r (reward) to a numpy array with the same data type as the first element of model's parameter list - done=np.array(done,self.nn.param[0].dtype.name) # Convert done (done flag) to a numpy array with the same data type as the first element of model's parameter list - else: # If the first element of model's parameter list is a list (multiple networks) - next_s=np.array(next_s,self.nn.param[0][0].dtype.name) # Convert next_s (next state) to a numpy array with the same data type as the first element of the first element of model's parameter list - r=np.array(r,self.nn.param[0][0].dtype.name) # Convert r (reward) to a numpy array with the same data type as the first element of the first element of model's parameter list - done=np.array(done,self.nn.param[0][0].dtype.name) # Convert done (done flag) to a numpy array with the same data type as the first element of the first element of model's parameter list - if hasattr(self.nn,'pool'): # If the model has a pool attribute - if hasattr(self.nn,'discriminator'): # If the model has a discriminator attribute - self.nn.pool(self.state_pool,self.action_pool,self.next_state_pool,self.reward_pool,self.done_pool,[s,a,next_s,reward,done]) # Call the model's pool method with pools and data as inputs - else: # If the model does not have a discriminator attribute - self.nn.pool(self.state_pool,self.action_pool,self.next_state_pool,self.reward_pool,self.done_pool,[s,a,next_s,r,done]) # Call the model's pool method with pools and data as inputs - else: # If the model does not have a pool attribute - self.pool(s,a,next_s,r,done) # Call the pool method with data as inputs - if hasattr(self.nn,'pr'): # If the model has a priority replay attribute - self.nn.pr.TD=np.append(self.nn.pr.TD,self.nn.initial_TD) # Append the initial TD value to the TD array of priority replay - if len(self.state_pool)>self.pool_size: # If the length of the state pool exceeds the maximum size of the pool - TD=np.array(0) # Create a zero array for TD value - self.nn.pr.TD=np.append(TD,self.nn.pr.TD[2:]) # Append TD value to the TD array of priority replay and remove the first two elements - self.reward=r+self.reward # Add r (reward) to total reward value - loss=self._train() # Call the _train method and get loss value - self.sc+=1 # Increase sc (step counter) by one - if done: # If done flag is True - if self.save_episode==True: # If save episode flag is True - episode=[s,a,next_s,r] # Set episode list to be equal to data - self.reward_list.append(self.reward) # Append total reward value to reward list - return loss,episode,done # Return loss value, episode list, and done flag - elif self.save_episode==True: # If save episode flag is True and done flag is False - episode=[s,a,next_s,r] # Set episode list to be equal to data - s=next_s # Set s (state) to be equal to next_s (next state) - self.reward_list.append(self.reward) # Append total reward value to reward list - return loss,episode,done # Return loss value, episode list, and done flag - - - def train(self,episode_count,save=None,one=True,p=None,s=None): # This is a method for training the network for multiple episodes - avg_reward=None # Initialize average reward value to None - if p==None: # If p value is None (p is used for printing frequency) - self.p=9 # Set it to be 9 (print every 10 episodes) - else: # If p value is given - self.p=p-1 # Set it to be p minus one (print every p episodes) - if s==None: # If s value is None (s is used for saving frequency) - self.s=1 # Set it to be 1 (save every episode) - self.file_list=None # Set file list to be None (no need to store file names) - else: # If s value is given - self.s=s-1 # Set it to be s minus one (save every s episodes) - self.file_list=[] # Initialize file list to empty list (need to store file names) - if episode_count!=None: # If episode count value is not None (there is a limit on episodes) - for i in range(episode_count): # For each episode in episode count value - t1=time.time() # Record the start time of the episode - loss,episode,done=self.train_() # Call the train_ method and get loss value, episode list, and done flag - if self.trial_count!=None: # If trial count value is not None (need to calculate average reward) - if len(self.reward_list)>=self.trial_count: # If the length of the reward list is greater than or equal to the trial count value - avg_reward=statistics.mean(self.reward_list[-self.trial_count:]) # Calculate the average reward by using the mean function on the last trial count elements of the reward list - if self.criterion!=None and avg_reward>=self.criterion: # If criterion value is not None and average reward is greater than or equal to criterion value (achieved the goal) - t2=time.time() # Record the end time of the episode - self.total_time+=(t2-t1) # Add the episode time to the total time - time_=self.total_time-int(self.total_time) # Get the decimal part of the total time - if time_<0.5: # If the decimal part is less than 0.5 - self.total_time=int(self.total_time) # Round down the total time - else: # If the decimal part is greater than or equal to 0.5 - self.total_time=int(self.total_time)+1 # Round up the total time - print('episode:{0}'.format(self.total_episode)) # Print the current episode number - print('last loss:{0:.6f}'.format(loss)) # Print the last loss value with six decimal places - print('average reward:{0}'.format(avg_reward)) # Print the average reward value - print() - print('time:{0}s'.format(self.total_time)) # Print the total time in seconds - return # Return nothing and exit the method - self.loss=loss # Set loss attribute to be equal to loss value - self.loss_list.append(loss) # Append loss value to loss list - self.total_episode+=1 # Increase total episode by one - if episode_count%10!=0: # If episode count is not divisible by 10 - p=episode_count-episode_count%self.p # Calculate p by subtracting the remainder of dividing episode count by p from episode count - p=int(p/self.p) # Divide p by p and round down - s=episode_count-episode_count%self.s # Calculate s by subtracting the remainder of dividing episode count by s from episode count - s=int(s/self.s) # Divide s by s and round down - else: # If episode count is divisible by 10 - p=episode_count/(self.p+1) # Divide episode count by p plus one and round down - p=int(p) # Convert p to integer - s=episode_count/(self.s+1) # Divide episode count by s plus one and round down - s=int(s) # Convert s to integer - if p==0: # If p is zero (should not happen) - p=1 # Set it to be one (print every episode) - if s==0: # If s is zero (should not happen) - s=1 # Set it to be one (save every episode) - if i%p==0: # If i (current episode number) is divisible by p (print frequency) - if len(self.state_pool)>=self.batch: # If the length of the state pool is greater than or equal to the batch size (enough data for training) - print('episode:{0} loss:{1:.6f}'.format(i+1,loss)) # Print the current episode number and loss value with six decimal places - if avg_reward!=None: # If average reward value is not None (calculated average reward) - print('episode:{0} average reward:{1}'.format(i+1,avg_reward)) # Print the current episode number and average reward value - else: # If average reward value is None (did not calculate average reward) - print('episode:{0} reward:{1}'.format(i+1,self.reward)) # Print the current episode number and total reward value - print() - if save!=None and i%s==0: # If save value is not None (need to save data) and i (current episode number) is divisible by s (save frequency) - self.save(self.total_episode,one) # Call the save method with total episode and one flag as inputs - if self.save_episode==True: # If save episode flag is True (need to save episodes) - if done: # If done flag is True (episode ended) - episode.append('done') # Append 'done' to episode list - self.episode_set.append(episode) # Append episode list to episode set list - if self.max_episode_count!=None and len(self.episode_set)>=self.max_episode_count: # If max episode count value is not None (there is a limit on episodes to save) and the length of episode set list is greater than or equal to max episode count value - self.save_episode=False # Set save episode flag to False (stop saving episodes) - try: # Try to execute the following code block - self.nn.ec.assign_add(1) # Add one to the model's episode counter using assign_add method - except Exception: # If an exception occurs - pass # Do nothing and ignore the exception - t2=time.time() # Record the end time of the episode - self.time+=(t2-t1) # Add the episode time to the time attribute - else: # If episode count value is None (no limit on episodes) - i=0 # Initialize i to zero - while True: # Loop indefinitely until break statement is executed - t1=time.time() # Record the start time of the episode - loss,episode,done=self.train_() # Call the train_ method and get loss value, episode list, and done flag - if self.trial_count!=None: # If trial count value is not None (need to calculate average reward) - if len(self.reward_list)==self.trial_count: # If the length of the reward list is equal to the trial count value - avg_reward=statistics.mean(self.reward_list[-self.trial_count:]) # Calculate the average reward by using the mean function on the last trial count elements of the reward list - if self.criterion!=None and avg_reward>=self.criterion: # If criterion value is not None and average reward is greater than or equal to criterion value (achieved the goal) - t2=time.time() # Record the end time of the episode - self.total_time+=(t2-t1) # Add the episode time to the total time - time_=self.total_time-int(self.total_time) # Get the decimal part of the total time - if time_<0.5: # If the decimal part is less than 0.5 - self.total_time=int(self.total_time) # Round down the total time - else: # If the decimal part is greater than or equal to 0.5 - self.total_time=int(self.total_time)+1 # Round up the total time - print('episode:{0}'.format(self.total_episode)) # Print the current episode number - print('last loss:{0:.6f}'.format(loss)) # Print the last loss value with six decimal places - print('average reward:{0}'.format(avg_reward)) # Print the average reward value - print() - print('time:{0}s'.format(self.total_time)) # Print the total time in seconds - return # Return nothing and exit the method - self.loss=loss # Set loss attribute to be equal to loss value - self.loss_list.append(loss) # Append loss value to loss list - i+=1 # Increase i by one - self.total_episode+=1 # Increase total episode by one - if episode_count%10!=0: # If episode count is not divisible by 10 - p=episode_count-episode_count%self.p # Calculate p by subtracting the remainder of dividing episode count by p from episode count - p=int(p/self.p) # Divide p by p and round down - s=episode_count-episode_count%self.s # Calculate s by subtracting the remainder of dividing episode count by s from episode count - s=int(s/self.s) # Divide s by s and round down - else: # If episode count is divisible by 10 - p=episode_count/(self.p+1) # Divide episode count by p plus one and round down - p=int(p) # Convert p to integer - s=episode_count/(self.s+1) # Divide episode count by s plus one and round down - s=int(s) # Convert s to integer - if i%p==0: # If i (current episode number) is divisible by p (print frequency) - if len(self.state_pool)>=self.batch: # If the length of the state pool is greater than or equal to the batch size (enough data for training) - print('episode:{0} loss:{1:.6f}'.format(i+1,loss)) # Print the current episode number and loss value with six decimal places - if avg_reward!=None: # If average reward value is not None (calculated average reward) - print('episode:{0} average reward:{1}'.format(i+1,avg_reward)) # Print the current episode number and average reward value - else: # If average reward value is None (did not calculate average reward) - print('episode:{0} reward:{1}'.format(i+1,self.reward)) # Print the current episode number and total reward value - print() - if save!=None and i%s==0: # If save value is not None (need to save data) and i (current episode number) is divisible by s (save frequency) - self.save(self.total_episode,one) # Call the save method with total episode and one flag as inputs - if self.save_episode==True: # If save episode flag is True (need to save episodes) - if done: # If done flag is True (episode ended) - episode.append('done') # Append 'done' to episode list - self.episode_set.append(episode) # Append episode list to episode set list - if self.max_episode_count!=None and len(self.episode_set)>=self.max_episode_count: # If max episode count value is not None (there is a limit on episodes to save) and the length of episode set list is greater than or equal to max episode count value - self.save_episode=False # Set save episode flag to False (stop saving episodes) - try: # Try to execute the following code block - self.nn.ec.assign_add(1) # Add one to the model's episode counter using assign_add method - except Exception: # If an exception occurs - self.nn.ec+=1 # Use normal addition instead - t2=time.time() # Record the end time of the episode - self.time+=(t2-t1) # Add the episode time to the time attribute - time_=self.time-int(self.time) # Get the decimal part of the time attribute - if time_<0.5: # If the decimal part is less than 0.5 - self.total_time=int(self.time) # Round down the time attribute - else: # If the decimal part is greater than or equal to 0.5 - self.total_time=int(self.time)+1 # Round up the time attribute - self.total_time+=self.time # Add the time attribute to the total time attribute - print('last loss:{0:.6f}'.format(loss)) # Print the last loss value with six decimal places - print('last reward:{0}'.format(self.reward)) # Print the last reward value - print() - print('time:{0}s'.format(self.time)) # Print the time attribute in seconds - return # Return nothing - - - def train_online(self): # This is a method for training the network online (without using a pool) - while True: # Loop indefinitely until break statement is executed - if hasattr(self.nn,'save'): # If the model has a save attribute - self.nn.save(self.save) # Call the model's save method with save attribute as input - if hasattr(self.nn,'stop_flag'): # If the model has a stop flag attribute - if self.nn.stop_flag==True: # If the stop flag is True (need to stop training) - return # Return nothing and exit the method - if hasattr(self.nn,'stop_func'): # If the model has a stop function attribute - if self.nn.stop_func(): # If the stop function returns True (need to stop training) - return # Return nothing and exit the method - if hasattr(self.nn,'suspend_func'): # If the model has a suspend function attribute - self.nn.suspend_func() # Call the suspend function to check if training needs to be suspended - data=self.nn.online() # Get online data by using model's online method - if data=='stop': # If data is 'stop' (need to stop training) - return # Return nothing and exit the method - elif data=='suspend': # If data is 'suspend' (need to suspend training) - self.nn.suspend_func() # Call the suspend function to suspend training - loss=self.opt_ol(data[0],data[1],data[2],data[3],data[4]) # Get loss value by using opt_ol method with data as inputs - loss=loss.numpy() # Convert loss value to numpy array - self.nn.train_loss_list.append(loss) # Append loss value to model's train loss list - if len(self.nn.train_acc_list)==self.nn.max_length: # If the length of model's train accuracy list is equal to model's max length value - del self.nn.train_acc_list[0] # Delete the first element of model's train accuracy list - if hasattr(self.nn,'counter'): # If the model has a counter attribute - self.nn.counter+=1 # Increase the counter by one - return # Return nothing diff --git a/Note 7.0 documentation/RL/kernel/parallel/kernel.py b/Note 7.0 documentation/RL/kernel/parallel/kernel.py deleted file mode 100644 index 14d92f97..00000000 --- a/Note 7.0 documentation/RL/kernel/parallel/kernel.py +++ /dev/null @@ -1,664 +0,0 @@ -import tensorflow as tf -from tensorflow.python.ops import state_ops -from tensorflow.python.util import nest -from multiprocessing import Value,Array -import numpy as np -import statistics - - -class kernel: - def __init__(self,nn=None,process=None): - self.nn=nn # the neural network model to be trained - if process!=None: # the number of processes to be used for parallel training - self.reward=np.zeros(process,dtype=np.float32) # the reward array for each process - self.sc=np.zeros(process,dtype=np.int32) # the step counter array for each process - self.epsilon=None # the epsilon value for epsilon-greedy policy - self.episode_step=None # the maximum number of steps for each episode - self.pool_size=None # the size of the memory pool for storing experience data - self.episode=None # the number of episodes for training - self.batch=None # the size of the batch for training - self.update_step=None # the frequency of updating the neural network parameters - self.trial_count=None # the number of trials to calculate the average reward - self.criterion=None # the criterion for stopping the training based on the average reward - self.process=process # the number of processes to be used for parallel training - self.PO=None # the parallel optimization mode (1, 2 or 3) - self.priority_flag=False # a flag indicating whether to use priority optimization or not - self.max_opt=None # the maximum number of optimization steps for priority optimization - self.stop=False # a flag indicating whether to stop the training or not - self.opt_counter=None # a counter array for tracking the optimization steps for each process - self.s=None # a state variable for online training - self.filename='save.dat' # a file name for saving the neural network parameters - - - def init(self,manager): - self.state_pool=manager.dict({}) # a shared dictionary for storing state data for each process - self.action_pool=manager.dict({}) # a shared dictionary for storing action data for each process - self.next_state_pool=manager.dict({}) # a shared dictionary for storing next state data for each process - self.reward_pool=manager.dict({}) # a shared dictionary for storing reward data for each process - self.done_pool=manager.dict({}) # a shared dictionary for storing done flag data for each process - self.reward=Array('f',self.reward) # a shared array for storing reward data for each process - if type(self.nn.param[0])!=list: - self.loss=np.zeros(self.process,dtype=self.nn.param[0].dtype.name) # a loss array for each process - else: - self.loss=np.zeros(self.process,dtype=self.nn.param[0][0].dtype.name) # a loss array for each process - self.loss=Array('f',self.loss) # a shared array for storing loss data for each process - self.sc=Array('i',self.sc) # a shared array for storing step counter data for each process - self.process_counter=Value('i',0) # a shared value for counting the number of running processes - self.probability_list=manager.list([]) # a shared list for storing probability data for each process - self.running_flag_list=manager.list([]) # a shared list for storing running flag data for each process - self.finish_list=manager.list([]) # a shared list for storing the finished process indices - self.running_flag=manager.list([0]) # a shared list for storing the running flag for each process - self.reward_list=manager.list([]) # a shared list for storing the total reward for each episode - self.loss_list=manager.list([]) # a shared list for storing the average loss for each episode - self.episode_counter=Value('i',0) # a shared value for counting the number of completed episodes - self.total_episode=Value('i',0) # a shared value for counting the total number of episodes - self.priority_p=Value('i',0) # a shared value for storing the priority process index - if self.priority_flag==True: # if priority optimization is enabled - self.opt_counter=Array('i',np.zeros(self.process,dtype=np.int32)) # a shared array for storing the optimization counter for each process - try: - self.nn.opt_counter=manager.list([self.nn.opt_counter]) # a shared list for storing the optimization counter for the neural network model - except Exception: - self.opt_counter_=manager.list() # a dummy list for handling exception - try: - self.nn.ec=manager.list([self.nn.ec]) # a shared list for storing the episode counter for the neural network model - except Exception: - self.ec_=manager.list() # a dummy list for handling exception - try: - self.nn.bc=manager.list([self.nn.bc]) # a shared list for storing the batch counter for the neural network model - except Exception: - self.bc_=manager.list() # a dummy list for handling exception - self.episode_=Value('i',self.total_episode.value) # a copy of the total episode value - self.stop_flag=Value('b',False) # a shared flag for indicating whether to stop the training or not - self.save_flag=Value('b',False) # a shared flag for indicating whether to save the neural network parameters or not - self.file_list=manager.list([]) # a shared list for storing the file names for saving the neural network parameters - self.param=manager.dict() # a shared dictionary for storing the neural network parameters - self.param[7]=self.nn.param # assign the neural network parameters to the dictionary with key 7 - return - - - def init_online(self,manager): - self.nn.train_loss_list=manager.list([]) # a shared list for storing the training loss data for online training - self.nn.counter=manager.list([]) # a shared list for storing the counter data for online training - self.nn.exception_list=manager.list([]) # a shared list for storing the exception data for online training - self.param=manager.dict() # a shared dictionary for storing the neural network parameters - self.param[7]=self.nn.param # assign the neural network parameters to the dictionary with key 7 - return - - - def action_vec(self): - self.action_one=np.ones(self.action_count,dtype=np.int8) # an array of ones with length equal to the action count - return - - - def set_up(self,epsilon=None,episode_step=None,pool_size=None,batch=None,update_step=None,trial_count=None,criterion=None): - if epsilon!=None: # if epsilon value is given - self.epsilon=np.ones(self.process)*epsilon # assign epsilon value to an array with length equal to the process count - self.action_vec() # call the action_vec method to create an action array - if episode_step!=None: # if episode step value is given - self.episode_step=episode_step # assign episode step value to an attribute - if pool_size!=None: # if pool size value is given - self.pool_size=pool_size # assign pool size value to an attribute - if batch!=None: # if batch size value is given - self.batch=batch # assign batch size value to an attribute - if update_step!=None: # if update step value is given - self.update_step=update_step # assign update step value to an attribute - if trial_count!=None: # if trial count value is given - self.trial_count=trial_count # assign trial count value to an attribute - if criterion!=None: # if criterion value is given - self.criterion=criterion # assign criterion value to an attribute - return - - - def epsilon_greedy_policy(self,s,epsilon): - action_prob=self.action_one*epsilon/len(self.action_one) # calculate the action probability based on epsilon and action array - best_a=np.argmax(self.nn.nn.fp(s)) # find the best action index based on the neural network output - action_prob[best_a]+=1-epsilon # increase the probability of the best action by 1-epsilon - return action_prob # return the action probability array - - - def pool(self,s,a,next_s,r,done,pool_lock,index): - pool_lock[index].acquire() # acquire the lock for the given index - try: - if type(self.state_pool[index])!=np.ndarray and self.state_pool[index]==None: # if the state pool for the given index is empty - self.state_pool[index]=s # assign the state data to the state pool - if type(a)==int: # if the action data is an integer - a=np.array(a) # convert it to a numpy array - self.action_pool[index]=np.expand_dims(a,axis=0) # add a dimension and assign it to the action pool - else: # if the action data is not an integer - self.action_pool[index]=a # assign it to the action pool directly - self.next_state_pool[index]=np.expand_dims(next_s,axis=0) # add a dimension and assign the next state data to the next state pool - self.reward_pool[index]=np.expand_dims(r,axis=0) # add a dimension and assign the reward data to the reward pool - self.done_pool[index]=np.expand_dims(done,axis=0) # add a dimension and assign the done flag data to the done pool - else: # if the state pool for the given index is not empty - try: - self.state_pool[index]=np.concatenate((self.state_pool[index],s),0) # concatenate the state data to the state pool - if type(a)==int: # if the action data is an integer - a=np.array(a) # convert it to a numpy array - self.action_pool[index]=np.concatenate((self.action_pool[index],np.expand_dims(a,axis=0)),0) # add a dimension and concatenate it to the action pool - else: # if the action data is not an integer - self.action_pool[index]=np.concatenate((self.action_pool[index],a),0) # concatenate it to the action pool directly - self.next_state_pool[index]=np.concatenate((self.next_state_pool[index],np.expand_dims(next_s,axis=0)),0) # add a dimension and concatenate the next state data to the next state pool - self.reward_pool[index]=np.concatenate((self.reward_pool[index],np.expand_dims(r,axis=0)),0) # add a dimension and concatenate the reward data to the reward pool - self.done_pool[index]=np.concatenate((self.done_pool[index],np.expand_dims(done,axis=0)),0) # add a dimension and concatenate the done flag data to the done pool - except Exception: - pass # ignore any exception that may occur during concatenation - if type(self.state_pool[index])==np.ndarray and len(self.state_pool[index])>self.pool_size: # if the state pool for the given index exceeds the pool size limit - self.state_pool[index]=self.state_pool[index][1:] # remove the oldest state data from the state pool - self.action_pool[index]=self.action_pool[index][1:] # remove the oldest action data from the action pool - self.next_state_pool[index]=self.next_state_pool[index][1:] # remove the oldest next state data from the next state pool - self.reward_pool[index]=self.reward_pool[index][1:] # remove the oldest reward data from the reward pool - self.done_pool[index]=self.done_pool[index][1:] # remove the oldest done flag data from the done pool - except Exception: - pool_lock[index].release() # release the lock for the given index in case of any exception - return - pool_lock[index].release() # release the lock for the given index after storing all data in pools - return - - - def get_index(self,p,lock): - while len(self.running_flag_list)self.process_counter.value: # if the running flag list for the given p is not consistent with the process counter value - self.running_flag_list[p]=self.running_flag[1:].copy() # update the running flag list for the given p with a copy of the running flag list without the first element - while len(self.probability_list)=self.trial_count: # if reward list has enough elements - avg_reward=statistics.mean(self.reward_list[-self.trial_count:]) # calculate the average reward of the last trial count elements - if self.criterion!=None and avg_reward>=self.criterion: # if criterion value is given and average reward is greater than or equal to criterion value - return True # return True, indicating that training should end - return False # otherwise, return False, indicating that training should continue - - - @tf.function(jit_compile=True) # use tensorflow function decorator with jit_compile option to improve computation efficiency and automatic differentiation - def opt(self,state_batch,action_batch,next_state_batch,reward_batch,done_batch,p,lock,g_lock=None): - with tf.GradientTape(persistent=True) as tape: # use tensorflow gradient tape with persistent option to record operations for automatic differentiation - try: - try: - loss=self.nn.loss(state_batch,action_batch,next_state_batch,reward_batch,done_batch) # calculate loss by calling loss attribute of neural network model with batch data as arguments - except Exception: - loss=self.nn.loss(state_batch,action_batch,next_state_batch,reward_batch,done_batch,p) # if exception occurs, try again with process index as an additional argument - except Exception as e: - raise e # if exception still occurs, raise it - if self.PO==1: # if parallel optimization mode is 1 - if self.priority_flag==True and self.priority_p.value!=-1: # if priority optimization is enabled and priority process index is not -1 - while True: - if self.stop_flag.value==True: # if stop flag is True - return None,None # return None values for loss and parameter - if p==self.priority_p.value: # if process index is equal to priority process index - break # break out of the loop - else: # otherwise - continue # continue looping - lock[0].acquire() # acquire lock with index 0 - if self.stop_func_(lock[0]): # call stop_func_ method with lock as argument and check if it returns True - return None,None # return None values for loss and parameter - if hasattr(self.nn,'gradient'): # if neural network model has an attribute called gradient - gradient=self.nn.gradient(tape,loss) # calculate gradient by calling gradient attribute of neural network model with tape and loss as arguments - else: # otherwise, assume that neural network model does not have an attribute called gradient - if hasattr(self.nn,'nn'): # if neural network model has an attribute called nn - gradient=tape.gradient(loss,self.nn.param) # calculate gradient by calling gradient method of tape with loss and neural network model parameter as arguments - else: # otherwise, assume that neither nn nor gradient attributes exist in neural network model - actor_gradient=tape.gradient(loss[0],self.nn.param[0]) # calculate actor gradient by calling gradient method of tape with actor loss and actor parameter as arguments - critic_gradient=tape.gradient(loss[1],self.nn.param[1]) # calculate critic gradient by calling gradient method of tape with critic loss and critic parameter as arguments - try: - if hasattr(self.nn,'attenuate'): # if neural network model has an attribute called attenuate - try: - gradient=self.nn.attenuate(gradient,self.nn.opt_counter,p) # attenuate gradient by calling attenuate attribute of neural network model with gradient, optimization counter and process index as arguments - except Exception: - actor_gradient=self.nn.attenuate(actor_gradient,self.nn.opt_counter,p) # if exception occurs, try to attenuate actor gradient by calling attenuate attribute of neural network model with actor gradient, optimization counter and process index as arguments - critic_gradient=self.nn.attenuate(critic_gradient,self.nn.opt_counter,p) # and try to attenuate critic gradient by calling attenuate attribute of neural network model with critic gradient, optimization counter and process index as arguments - except Exception as e: - raise e # if exception still occurs, raise it - try: - try: - param=self.nn.opt(gradient,p) # optimize parameter by calling opt attribute of neural network model with gradient and process index as arguments - except Exception: - param=self.nn.opt(gradient) # if exception occurs, try again without process index as argument - except Exception as e: - raise e # if exception still occurs, raise it - lock[0].release() # release lock with index 0 - elif self.PO==2: # if parallel optimization mode is 2 - g_lock.acquire() # acquire global lock - if self.stop_func_(g_lock): # call stop_func_ method with global lock as argument and check if it returns True - return None,None # return None values for loss and parameter - if hasattr(self.nn,'gradient'): # if neural network model has an attribute called gradient - gradient=self.nn.gradient(tape,loss) # calculate gradient by calling gradient attribute of neural network model with tape and loss as arguments - else: # otherwise, assume that neural network model does not have an attribute called gradient - if hasattr(self.nn,'nn'): # if neural network model has an attribute called nn - gradient=tape.gradient(loss,self.nn.param) # calculate gradient by calling gradient method of tape with loss and neural network model parameter as arguments - else: # otherwise, assume that neither nn nor gradient attributes exist in neural network model - actor_gradient=tape.gradient(loss[0],self.nn.param[0]) # calculate actor gradient by calling gradient method of tape with actor loss and actor parameter as arguments - critic_gradient=tape.gradient(loss[1],self.nn.param[1]) # calculate critic gradient by calling gradient method of tape with critic loss and critic parameter as arguments - g_lock.release() # release global lock - if self.priority_flag==True and self.priority_p.value!=-1: # if priority optimization is enabled and priority process index is not -1 - while True: - if self.stop_flag.value==True: # if stop flag is True - return None,None # return None values for loss and parameter - if p==self.priority_p.value: # if process index is equal to priority process index - break # break out of the loop - else: # otherwise - continue # continue looping - lock[0].acquire() # acquire lock with index 0 - try: - if hasattr(self.nn,'attenuate'): # if neural network model has an attribute called attenuate - try: - gradient=self.nn.attenuate(gradient,self.nn.opt_counter,p) # attenuate gradient by calling attenuate attribute of neural network model with gradient, optimization counter and process index as arguments - except Exception: - actor_gradient=self.nn.attenuate(actor_gradient,self.nn.opt_counter,p) # if exception occurs, try to attenuate actor gradient by calling attenuate attribute of neural network model with actor gradient, optimization counter and process index as arguments - critic_gradient=self.nn.attenuate(critic_gradient,self.nn.opt_counter,p) # and try to attenuate critic gradient by calling attenuate attribute of neural network model with critic gradient, optimization counter and process index as arguments - except Exception as e: - raise e # if exception still occurs, raise it - try: - try: - param=self.nn.opt(gradient,p) # optimize parameter by calling opt attribute of neural network model with gradient and process index as arguments - except Exception: - param=self.nn.opt(gradient) # if exception occurs, try again without process index as argument - except Exception as e: - raise e # if exception still occurs, raise it - lock[0].release() # release lock with index 0 - elif self.PO==3: # if parallel optimization mode is 3 - if self.priority_flag==True and self.priority_p.value!=-1: # if priority optimization is enabled and priority process index is not -1 - while True: - if self.stop_flag.value==True: # if stop flag is True - return None,None # return None values for loss and parameter - if p==self.priority_p.value: # if process index is equal to priority process index - break # break out of the loop - else: # otherwise - continue # continue looping - if self.stop_func_(): # call stop_func_ method without lock as argument and check if it returns True - return None,None # return None values for loss and parameter - if hasattr(self.nn,'gradient'): # if neural network model has an attribute called gradient - gradient=self.nn.gradient(tape,loss) # calculate gradient by calling gradient attribute of neural network model with tape and loss as arguments - else: # otherwise, assume that neural network model does not have an attribute called gradient - if hasattr(self.nn,'nn'): # if neural network model has an attribute called nn - gradient=tape.gradient(loss,self.nn.param) # calculate gradient by calling gradient method of tape with loss and neural network model parameter as arguments - else: # otherwise, assume that neither nn nor gradient attributes exist in neural network model - actor_gradient=tape.gradient(loss[0],self.nn.param[0]) # calculate actor gradient by calling gradient method of tape with actor loss and actor parameter as arguments - critic_gradient=tape.gradient(loss[1],self.nn.param[1]) # calculate critic gradient by calling gradient method of tape with critic loss and critic parameter as arguments - try: - if hasattr(self.nn,'attenuate'): # if neural network model has an attribute called attenuate - try: - gradient=self.nn.attenuate(gradient,self.nn.opt_counter,p) # attenuate gradient by calling attenuate attribute of neural network model with gradient, optimization counter and process index as arguments - except Exception: - actor_gradient=self.nn.attenuate(actor_gradient,self.nn.opt_counter,p) # if exception occurs, try to attenuate actor gradient by calling attenuate attribute of neural network model with actor gradient, optimization counter and process index as arguments - critic_gradient=self.nn.attenuate(critic_gradient,self.nn.opt_counter,p) # and try to attenuate critic gradient by calling attenuate attribute of neural network model with critic gradient, optimization counter and process index as arguments - except Exception as e: - raise e # if exception still occurs, raise it - try: - try: - param=self.nn.opt(gradient,p) # optimize parameter by calling opt attribute of neural network model with gradient and process index as arguments - except Exception: - param=self.nn.opt(gradient) # if exception occurs, try again without process index as argument - except Exception as e: - raise e # if exception still occurs, raise it - return loss,param # return loss and parameter - - - def update_nn_param(self,param=None): - if param==None: # if parameter is not given - parameter_flat=nest.flatten(self.nn.param) # flatten the neural network model parameter to a list - parameter7_flat=nest.flatten(self.param[7]) # flatten the parameter with key 7 in the shared dictionary to a list - else: # otherwise, assume that parameter is given - parameter_flat=nest.flatten(self.nn.param) # flatten the neural network model parameter to a list - parameter7_flat=nest.flatten(param) # flatten the given parameter to a list - for i in range(len(parameter_flat)): # loop through the indices of the flattened lists - if param==None: # if parameter is not given - state_ops.assign(parameter_flat[i],parameter7_flat[i]) # assign the value of the parameter with key 7 in the shared dictionary to the neural network model parameter - else: # otherwise, assume that parameter is given - state_ops.assign(parameter_flat[i],parameter7_flat[i]) # assign the value of the given parameter to the neural network model parameter - self.nn.param=nest.pack_sequence_as(self.nn.param,parameter_flat) # pack the flattened list back to the original structure of the neural network model parameter - self.param[7]=nest.pack_sequence_as(self.param[7],parameter7_flat) # pack the flattened list back to the original structure of the parameter with key 7 in the shared dictionary - return - - - def _train(self,p,j,batches,length,lock,g_lock): - if j==batches-1: # if it is the last batch - index1=batches*self.batch # calculate the start index of the batch - index2=self.batch-(length-batches*self.batch) # calculate the end index of the batch - state_batch=np.concatenate((self.state_pool[p][index1:length],self.state_pool[p][:index2]),0) # concatenate state data from two parts of the state pool to form a complete batch - action_batch=np.concatenate((self.action_pool[p][index1:length],self.action_pool[p][:index2]),0) # concatenate action data from two parts of the action pool to form a complete batch - next_state_batch=np.concatenate((self.next_state_pool[p][index1:length],self.next_state_pool[p][:index2]),0) # concatenate next state data from two parts of the next state pool to form a complete batch - reward_batch=np.concatenate((self.reward_pool[p][index1:length],self.reward_pool[p][:index2]),0) # concatenate reward data from two parts of the reward pool to form a complete batch - done_batch=np.concatenate((self.done_pool[p][index1:length],self.done_pool[p][:index2]),0) # concatenate done flag data from two parts of the done pool to form a complete batch - if self.PO==2: # if parallel optimization mode is 2 - if type(g_lock)!=list: # if global lock is not a list - pass # do nothing - elif len(g_lock)==self.process: # if global lock is a list with length equal to process count - ln=p # set local lock index to process index - g_lock=g_lock[ln] # set global lock to local lock with index ln - else: # otherwise, assume that global lock is a list with length not equal to process count - ln=int(np.random.choice(len(g_lock))) # randomly choose a local lock index from global lock list - g_lock=g_lock[ln] # set global lock to local lock with index ln - loss,param=self.opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch,p,lock,g_lock) # call opt method with batch data, process index, lock and global lock as arguments and get loss and parameter as outputs - else: # otherwise, assume that parallel optimization mode is not 2 - loss,param=self.opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch,p,lock) # call opt method with batch data, process index and lock as arguments and get loss and parameter as outputs - if self.stop_flag.value==True: # if stop flag is True - return # return without doing anything - self.param[7]=param # assign parameter to parameter with key 7 in shared dictionary - self.loss[p]+=loss # add loss to loss array for process index p - if hasattr(self.nn,'bc'): # if neural network model has an attribute called bc (batch counter) - bc=self.nn.bc[0] # get bc value from neural network model - bc.assign_add(1) # increment bc value by 1 - self.nn.bc[0]=bc # assign bc value back to neural network - else: # otherwise, assume that it is not the last batch - index1=j*self.batch # calculate the start index of the batch - index2=(j+1)*self.batch # calculate the end index of the batch - state_batch=self.state_pool[p][index1:index2] # get state data from state pool with start and end indices - action_batch=self.action_pool[p][index1:index2] # get action data from action pool with start and end indices - next_state_batch=self.next_state_pool[p][index1:index2] # get next state data from next state pool with start and end indices - reward_batch=self.reward_pool[p][index1:index2] # get reward data from reward pool with start and end indices - done_batch=self.done_pool[p][index1:index2] # get done flag data from done pool with start and end indices - if self.PO==2: # if parallel optimization mode is 2 - if type(g_lock)!=list: # if global lock is not a list - pass # do nothing - elif len(g_lock)==self.process: # if global lock is a list with length equal to process count - ln=p # set local lock index to process index - g_lock=g_lock[ln] # set global lock to local lock with index ln - else: # otherwise, assume that global lock is a list with length not equal to process count - ln=int(np.random.choice(len(g_lock))) # randomly choose a local lock index from global lock list - g_lock=g_lock[ln] # set global lock to local lock with index ln - loss,param=self.opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch,p,lock,g_lock) # call opt method with batch data, process index, lock and global lock as arguments and get loss and parameter as outputs - else: # otherwise, assume that parallel optimization mode is not 2 - loss,param=self.opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch,p,lock) # call opt method with batch data, process index and lock as arguments and get loss and parameter as outputs - if self.stop_flag.value==True: # if stop flag is True - return # return without doing anything - self.param[7]=param # assign parameter to parameter with key 7 in shared dictionary - self.loss[p]+=loss # add loss to loss array for process index p - if hasattr(self.nn,'bc'): # if neural network model has an attribute called bc (batch counter) - bc=self.nn.bc[0] # get bc value from neural network model - bc.assign_add(1) # increment bc value by 1 - self.nn.bc[0]=bc # assign bc value back to neural network model - return # return without any value - - - def train_(self,p,lock,g_lock): - if len(self.done_pool[p])=self.max_opt: # if maximum optimization step value is given and optimization counter value for priority process index is greater than or equal to it - self.priority_p.value=int(self.priority_p.value) # convert priority process index to integer type - elif self.max_opt==None: # otherwise, assume that maximum optimization step value is not given - self.priority_p.value=int(self.priority_p.value) # convert priority process index to integer type - else: # otherwise, assume that maximum optimization step value is given but optimization counter value for priority process index is less than it - self.priority_p.value=-1 # set priority process index to -1, meaning no priority process - if self.priority_flag==True: # if priority optimization is enabled - self.opt_counter[p]=0 # reset optimization counter value for process index p to zero - if hasattr(self.nn,'attenuate'): # if neural network model has an attribute called attenuate - opt_counter=self.nn.opt_counter[0] # get opt_counter value from neural network model - opt_counter.scatter_update(tf.IndexedSlices(0,p)) # update opt_counter value with 0 at process index p - self.nn.opt_counter[0]=opt_counter # assign opt_counter value back to neural network model - self._train(p,j,batches,length,lock,g_lock) # call _train method with process index, batch index, batch number, length, lock and global lock as arguments - if self.stop_flag.value==True: # if stop flag is True - return # return without doing anything - if self.priority_flag==True: # if priority optimization is enabled - opt_counter=np.frombuffer(self.opt_counter.get_obj(),dtype='i') # get optimization counter array from shared memory - opt_counter+=1 # increment optimization counter array by 1 - if hasattr(self.nn,'attenuate'): # if neural network model has an attribute called attenuate - opt_counter=self.nn.opt_counter[0] # get opt_counter value from neural network model - opt_counter.assign(opt_counter+1) # increment opt_counter value by 1 - self.nn.opt_counter[0]=opt_counter # assign opt_counter value back to neural network model - if self.PO==1 or self.PO==2: # if parallel optimization mode is 1 or 2 - lock[1].acquire() # acquire lock with index 1 - if self.update_step!=None: # if update step value is given - if self.sc[p]%self.update_step==0: # if step counter value for process index p is divisible by update step value - self.nn.update_param() # call update_param method of neural network model to update its parameters - else: # otherwise, assume that update step value is not given - self.nn.update_param() # call update_param method of neural network model to update its parameters - if self.PO==1 or self.PO==2: # if parallel optimization mode is 1 or 2 - lock[1].release() # release lock with index 1 - self.loss[p]=self.loss[p]/batches # calculate the average loss for process index p by dividing the total loss by the number of batches - self.sc[p]+=1 # increment the step counter value for process index p by 1 - if hasattr(self.nn,'ec'): # if neural network model has an attribute called ec (episode counter) - ec=self.nn.ec[0] # get ec value from neural network model - ec.assign_add(1) # increment ec value by 1 - self.nn.ec[0]=ec # assign ec value back to neural network model - return # return without any value - - - def train(self,p,lock,pool_lock,g_lock=None): - if self.PO==1 or self.PO==2: # if parallel optimization mode is 1 or 2 - lock[1].acquire() # acquire lock with index 1 - elif self.PO==3: # if parallel optimization mode is 3 - lock[1].acquire() # acquire lock with index 1 - self.state_pool[p]=None # set state pool for process index p to None - self.action_pool[p]=None # set action pool for process index p to None - self.next_state_pool[p]=None # set next state pool for process index p to None - self.reward_pool[p]=None # set reward pool for process index p to None - self.done_pool[p]=None # set done pool for process index p to None - self.running_flag.append(1) # append 1 to running flag list, indicating that process is running - self.process_counter.value+=1 # increment process counter value by 1, indicating that one more process is running - self.finish_list.append(None) # append None to finish list, indicating that process is not finished yet - if self.PO==1 or self.PO==2: # if parallel optimization mode is 1 or 2 - lock[1].release() # release lock with index 1 - elif self.PO==3: # if parallel optimization mode is 3 - lock[1].release() # release lock with index 1 - try: - epsilon=self.epsilon[p] # get epsilon value for process index p from epsilon array - except Exception: - epsilon=None # if exception occurs, set epsilon value to None - while True: # loop indefinitely until break condition is met - if self.stop_flag.value==True: # if stop flag is True - break # break out of the loop - if self.episode!=None and self.episode_counter.value>=self.episode: # if episode counter value is greater than or equal to episode - break # break out of the loop - s=self.nn.env(p=p,initial=True) # get initial state from calling env attribute of neural network model with process index and initial flag as arguments - if type(self.nn.param[0])!=list: # if neural network model parameter is not a list - s=np.array(s,self.nn.param[0].dtype.name) # convert state data to the same data type as the neural network model parameter - else: # otherwise, assume that neural network model parameter is a list - s=np.array(s,self.nn.param[0][0].dtype.name) # convert state data to the same data type as the first element of the neural network model parameter list - if self.episode_step==None: # if episode step value is not given - while True: # loop indefinitely until break condition is met - if self.episode!=None and self.episode_counter.value>=self.episode: # if episode counter value is greater than or equal to episode - break # break out of the loop - next_s,r,done,index=self.env(s,epsilon,p,lock,pool_lock) # get next state, reward, done flag and index from calling env method with state, epsilon, process index, lock and pool lock as arguments - self.reward[p]+=r # add reward to reward array for process index p - s=next_s # set state to next state - if type(self.done_pool[p])==np.ndarray: # if done pool for process index p is a numpy array - self.train_(p,lock,g_lock) # call train_ method with process index, lock and global lock as arguments - if self.stop_flag.value==True: # if stop flag is True - break # break out of the loop - if done: # if done flag is True - if self.PO==1 or self.PO==2: # if parallel optimization mode is 1 or 2 - lock[1].acquire() # acquire lock with index 1 - elif len(lock)==4: # otherwise, if lock has length 4 - lock[3].acquire() # acquire lock with index 3 - self.episode_counter.value+=1 # increment episode counter value by 1, indicating that one more episode is completed - self.total_episode.value+=1 # increment total episode value by 1, indicating that one more episode is completed - self.loss_list.append(self.loss[p]) # append loss value for process index p to loss list - if self.PO==1 or self.PO==2: # if parallel optimization mode is 1 or 2 - lock[1].release() # release lock with index 1 - elif len(lock)==4: # otherwise, if lock has length 4 - lock[3].release() # release lock with index 3 - break # break out of the loop - else: # otherwise, assume that episode step value is given - for l in range(self.episode_step): # loop through the episode step indices - if self.episode!=None and self.episode_counter.value>=self.episode: # if episode counter value is greater than or equal to episode - break # break out of the loop - next_s,r,done,index=self.env(s,epsilon,p,lock,pool_lock) # get next state, reward, done flag and index from calling env method with state, epsilon, process index, lock and pool lock as arguments - self.reward[p]+=r # add reward to reward array for process index p - s=next_s # set state to next state - if type(self.done_pool[p])==np.ndarray: # if done pool for process index p is a numpy array - self.train_(p,lock,g_lock) # call train_ method with process index, lock and global lock as arguments - if self.stop_flag.value==True: # if stop flag is True - break # break out of the loop - if done: # if done flag is True - if self.PO==1 or self.PO==2: # if parallel optimization mode is 1 or 2 - lock[1].acquire() # acquire lock with index 1 - elif len(lock)==4: # otherwise, if lock has length 4 - lock[3].acquire() # acquire lock with index 3 - self.episode_counter.value+=1 # increment episode counter value by 1, indicating that one more episode is completed - self.total_episode.value+=1 # increment total episode value by 1, indicating that one more episode is completed - self.loss_list.append(self.loss[p]) # append loss value for process index p to loss list - if self.PO==1 or self.PO==2: # if parallel optimization mode is 1 or 2 - lock[1].release() # release lock with index 1 - elif len(lock)==4: # otherwise, if lock has length 4 - lock[3].release() # release lock with index 3 - break # break out of the loop - if l==self.episode_step-1: # if it is the last episode step - if self.PO==1 or self.PO==2: # if parallel optimization mode is 1 or 2 - lock[1].acquire() # acquire lock with index 1 - elif len(lock)==4: # otherwise, if lock has length 4 - lock[3].acquire() # acquire lock with index 3 - self.episode_counter.value+=1 # increment episode counter value by 1, indicating that one more episode is completed - self.total_episode.value+=1 # increment total episode value by 1, indicating that one more episode is completed - self.loss_list.append(self.loss[p]) # append loss value for process index p to loss list - if self.PO==1 or self.PO==2: # if parallel optimization mode is 1 or 2 - lock[1].release() # release lock with index 1 - elif len(lock)==4: # otherwise, if lock has length 4 - lock[3].release() # release lock with index 3 - if self.PO==1 or self.PO==2: # if parallel optimization mode is 1 or 2 - lock[1].acquire() # acquire lock with index 1 - elif len(lock)==3 or len(lock)==4: # otherwise, if lock has length 3 or 4 - lock[2].acquire() # acquire lock with index 2 - self.save_() # call save_ method to save neural network parameters to file - self.reward_list.append(self.reward[p]) # append reward value for process index p to reward list - self.reward[p]=0 # reset reward value for process index p to zero - if self.PO==1 or self.PO==2: # if parallel optimization mode is 1 or 2 - lock[1].release() # release lock with index 1 - elif len(lock)==3 or len(lock)==4: # otherwise, if lock has length 3 or 4 - lock[2].release() # release lock with index 2 - self.running_flag[p+1]=0 # set running flag value for process index p plus one to zero, indicating that process is not running anymore - if p not in self.finish_list: # if process index p is not in finish list - self.finish_list[p]=p # assign process index p to finish list at process index p, indicating that process is finished - if self.PO==1 or self.PO==2: # if parallel optimization mode is 1 or 2 - lock[1].acquire() # acquire lock with index 1 - elif self.PO==3: # if parallel optimization mode is 3 - lock[1].acquire() # acquire lock with index 1 - self.process_counter.value-=1 # decrement process counter value by 1, indicating that one less process is running - if self.PO==1 or self.PO==2: # if parallel optimization mode is 1 or 2 - lock[1].release() # release lock with index 1 - elif self.PO==3: # if parallel optimization mode is 3 - lock[1].release() # release lock with index 1 - del self.state_pool[p] # delete state pool for process index p - del self.action_pool[p] # delete action pool for process index p - del self.next_state_pool[p] # delete next state pool for process index p - del self.reward_pool[p] # delete reward pool for process index p - del self.done_pool[p] # delete done pool for process index p - return # return without any value - - - def train_online(self,p,lock=None,g_lock=None): - if hasattr(self.nn,'counter'): # if neural network model has an attribute called counter - self.nn.counter.append(0) # append zero to counter list of neural network model - while True: # loop indefinitely until break condition is met - if hasattr(self.nn,'save'): # if neural network model has an attribute called save - self.nn.save(self.save,p) # call save attribute of neural network model with save and process index as arguments - if hasattr(self.nn,'stop_flag'): # if neural network model has an attribute called stop_flag - if self.nn.stop_flag==True: # if stop_flag value of neural network model is True - return # return without doing anything - if hasattr(self.nn,'stop_func'): # if neural network model has an attribute called stop_func - if self.nn.stop_func(p): # call stop_func attribute of neural network model with process index as argument and check if it returns True - return # return without doing anything - if hasattr(self.nn,'suspend_func'): # if neural network model has an attribute called suspend_func - self.nn.suspend_func(p) # call suspend_func attribute of neural network model with process index as argument - try: - data=self.nn.online(p) # get data from calling online attribute of neural network model with process index as argument - except Exception as e: - self.nn.exception_list[p]=e # if exception occurs, assign it to exception list of neural network model at process index p - if data=='stop': # if data is 'stop' - return # return without doing anything - elif data=='suspend': # if data is 'suspend' - self.nn.suspend_func(p) # call suspend_func attribute of neural network model with process index as argument - try: - if self.PO==2: # if parallel optimization mode is 2 - if type(g_lock)!=list: # if global lock is not a list - pass # do nothing - elif len(g_lock)==self.process: # if global lock is a list with length equal to process count - ln=p # set local lock index to process index - g_lock=g_lock[ln] # set global lock to local lock with index ln - else: # otherwise, assume that global lock is a list with length not equal to process count - ln=int(np.random.choice(len(g_lock))) # randomly choose a local lock index from global lock list - g_lock=g_lock[ln] # set global lock to local lock with index ln - loss,param=self.opt(data[0],data[1],data[2],data[3],data[4],p,lock,g_lock) # call opt method with data, process index, lock and global lock as arguments and get loss and parameter as outputs - else: # otherwise, assume that parallel optimization mode is not 2 - loss,param=self.opt(data[0],data[1],data[2],data[3],data[4],p,lock) # call opt method with data, process index and lock as arguments and get loss and parameter as outputs - except Exception as e: - if self.PO==1: # if parallel optimization mode is 1 - if lock[0].acquire(False): # try to acquire lock with index 0 without blocking - lock[0].release() # release lock with index 0 - elif self.PO==2: # if parallel optimization mode is 2 - if g_lock.acquire(False): # try to acquire global lock without blocking - g_lock.release() # release global lock - if lock[0].acquire(False): # try to acquire lock with index 0 without blocking - lock[0].release() # release lock with index 0 - self.nn.exception_list[p]=e # assign exception to exception list of neural network model at process index p - loss=loss.numpy() # convert loss to numpy array - if len(self.nn.train_loss_list)==self.nn.max_length: # if train loss list of neural network model has reached the maximum length - del self.nn.train_loss_list[0] # delete the first element of train loss list of neural network model - self.nn.train_loss_list.append(loss) # append loss to train loss list of neural network model - try: - if hasattr(self.nn,'counter'): # if neural network model has an attribute called counter - count=self.nn.counter[p] # get count value from counter list of neural network model at process index p - count+=1 # increment count value by 1 - self.nn.counter[p]=count # assign count value back to counter list of neural network model at process index p - except IndexError: - self.nn.counter.append(0) # if index error occurs, append zero to counter list of neural network model - count=self.nn.counter[p] # get count value from counter list of neural network model at process index p - count+=1 # increment count value by 1 - self.nn.counter[p]=count # assign count value back to counter list of neural network model at process index p - return # return without any value diff --git a/Note 7.0 documentation/RL/kernel/parallel/kernel_pytorch.py b/Note 7.0 documentation/RL/kernel/parallel/kernel_pytorch.py deleted file mode 100644 index b8fbd3ba..00000000 --- a/Note 7.0 documentation/RL/kernel/parallel/kernel_pytorch.py +++ /dev/null @@ -1,432 +0,0 @@ -import torch -from multiprocessing import Value,Array -from Note.nn.parallel.assign_device_pytorch import assign_device -import numpy as np -import statistics - - -class kernel: - def __init__(self,nn=None,process=None,device='GPU'): - self.nn=nn # the neural network to be trained - if process!=None: - self.reward=np.zeros(process,dtype=np.float32) # the reward array for each process - self.sc=np.zeros(process,dtype=np.int32) # the step counter array for each process - self.device=device # the device to use, GPU or CPU - self.epsilon=None # the epsilon value for epsilon-greedy policy - self.episode_step=None # the maximum number of steps per episode - self.pool_size=None # the size of the pool to store state, action, reward and done - self.episode=None # the maximum number of episodes to run - self.batch=None # the batch size for training - self.update_step=None # the update step for target network - self.trial_count=None # the number of trials to calculate average reward - self.process=process # the number of processes to run in parallel - self.priority_flag=False # the flag to indicate whether to use priority-based optimization - self.max_opt=None # the maximum number of optimization steps per process - self.stop=False # the flag to indicate whether to stop training - self.s=None # the state variable for online training - self.filename='save.dat' # the file name to save the model - - - def init(self,manager): - self.state_pool=manager.dict({}) # a dictionary to store state pool for each process - self.action_pool=manager.dict({}) # a dictionary to store action pool for each process - self.next_state_pool=manager.dict({}) # a dictionary to store next state pool for each process - self.reward_pool=manager.dict({}) # a dictionary to store reward pool for each process - self.done_pool=manager.dict({}) # a dictionary to store done pool for each process - self.reward=Array('f',self.reward) # a shared array to store reward for each process - self.loss=np.zeros(self.process) # an array to store loss for each process - self.loss=Array('f',self.loss) # a shared array to store loss for each process - self.sc=Array('i',self.sc) # a shared array to store step counter for each process - self.process_counter=Value('i',0) # a shared value to count the number of running processes - self.probability_list=manager.list([]) # a list to store probability distribution for selecting index from pool - self.running_flag_list=manager.list([]) # a list to store running flag list for each process - self.finish_list=manager.list([]) # a list to store finished processes' indices - self.running_flag=manager.list([0]) # a list to store running flag for each process, initialized with 0 - self.reward_list=manager.list([]) # a list to store reward history - self.loss_list=manager.list([]) # a list to store loss history - self.episode_counter=Value('i',0) # a shared value to count the number of episodes completed by all processes - self.total_episode=Value('i',0) # a shared value to count the total number of episodes completed by all processes - self.priority_p=Value('i',0) # a shared value to indicate which process has priority for optimization - if self.priority_flag==True: - self.opt_counter=Array('i',np.zeros(self.process,dtype=np.int32)) # an array to count optimization steps for each process - try: - self.nn.opt_counter=manager.list([self.nn.opt_counter]) # a list to store optimization counter for each neural network - except Exception: - self.opt_counter_=manager.list() # an empty list if no optimization counter is available - try: - self.nn.ec=manager.list([self.nn.ec]) # a list to store episode counter for each neural network - except Exception: - self.ec_=manager.list() # an empty list if no episode counter is available - try: - self.nn.bc=manager.list([self.nn.bc]) # a list to store batch counter for each neural network - except Exception: - self.bc_=manager.list() # an empty list if no batch counter is available - self.episode_=Value('i',self.total_episode.value) # a shared value to copy total episode value - self.stop_flag=Value('b',False) # a shared value to indicate whether to stop training - self.save_flag=Value('b',False) # a shared value to indicate whether to save the model - self.file_list=manager.list([]) # a list to store file names for saving the model - return - - - def init_online(self,manager): - self.nn.train_loss_list=manager.list([]) # a list to store training loss for online training - self.nn.counter=manager.list([]) # a list to store counter for online training - self.nn.exception_list=manager.list([]) # a list to store exceptions for online training - return - - - def action_vec(self): - self.action_one=np.ones(self.action_count,dtype=np.int8) # an array of ones with the same size as action space - return - - - def set_up(self,epsilon=None,episode_step=None,pool_size=None,batch=None,update_step=None,trial_count=None,criterion=None): - if epsilon!=None: - self.epsilon=np.ones(self.process)*epsilon # an array of epsilon values for each process - self.action_vec() # create the action vector - if episode_step!=None: - self.episode_step=episode_step # set the maximum number of steps per episode - if pool_size!=None: - self.pool_size=pool_size # set the size of the pool - if batch!=None: - self.batch=batch # set the batch size for training - if update_step!=None: - self.update_step=update_step # set the update step for target network - if trial_count!=None: - self.trial_count=trial_count # set the number of trials to calculate average reward - if criterion!=None: - self.criterion=criterion # set the criterion for stopping training based on average reward - return - - - def epsilon_greedy_policy(self,s,epsilon,p): - action_prob=self.action_one*epsilon/len(self.action_one) # create a uniform probability distribution for actions - s=torch.tensor(s,dtype=torch.float).to(assign_device(p,self.device)) # convert state to tensor and assign device - best_a=self.nn.nn(s).argmax() # get the best action from neural network output - action_prob[best_a.numpy()]+=1-epsilon # increase the probability of best action by (1-epsilon) - return action_prob - - - def pool(self,s,a,next_s,r,done,pool_lock,index): - pool_lock[index].acquire() # acquire lock for index - try: - if type(self.state_pool[index])!=np.ndarray and self.state_pool[index]==None: # if state pool is empty - self.state_pool[index]=s # store state - if type(a)==int: - a=np.array(a) - self.action_pool[index]=np.expand_dims(a,axis=0) # store action with one dimension - else: - self.action_pool[index]=a # store action - self.next_state_pool[index]=np.expand_dims(next_s,axis=0) # store next state with one dimension - self.reward_pool[index]=np.expand_dims(r,axis=0) # store reward with one dimension - self.done_pool[index]=np.expand_dims(done,axis=0) # store done with one dimension - else: - try: - self.state_pool[index]=np.concatenate((self.state_pool[index],s),0) # append state to state pool - if type(a)==int: - a=np.array(a) - self.action_pool[index]=np.concatenate((self.action_pool[index],np.expand_dims(a,axis=0)),0) # append action to action pool with one dimension - else: - self.action_pool[index]=np.concatenate((self.action_pool[index],a),0) # append action to action pool - self.next_state_pool[index]=np.concatenate((self.next_state_pool[index],np.expand_dims(next_s,axis=0)),0) # append next state to next state pool with one dimension - self.reward_pool[index]=np.concatenate((self.reward_pool[index],np.expand_dims(r,axis=0)),0) # append reward to reward pool with one dimension - self.done_pool[index]=np.concatenate((self.done_pool[index],np.expand_dims(done,axis=0)),0) # append done to done pool with one dimension - except Exception: - pass - if type(self.state_pool[index])==np.ndarray and len(self.state_pool[index])>self.pool_size: # if state pool exceeds the size limit - self.state_pool[index]=self.state_pool[index][1:] # remove the oldest state from state pool - self.action_pool[index]=self.action_pool[index][1:] # remove the oldest action from action pool - self.next_state_pool[index]=self.next_state_pool[index][1:] # remove the oldest next state from next state pool - self.reward_pool[index]=self.reward_pool[index][1:] # remove the oldest reward from reward pool - self.done_pool[index]=self.done_pool[index][1:] # remove the oldest done from done pool - except Exception: - pool_lock[index].release() # release lock for index if there is an exception - return - pool_lock[index].release() # release lock for index after updating the pools - return - - - def get_index(self,p,lock): - while len(self.running_flag_list)self.process_counter.value: # if running flag list has less elements than process counter or more elements than process counter - self.running_flag_list[p]=self.running_flag[1:].copy() # copy the running flag list and assign it to the p-th element of the list of lists - while len(self.probability_list)=self.trial_count: # if reward list has enough elements - avg_reward=statistics.mean(self.reward_list[-self.trial_count:]) # calculate the average reward of the last trial count elements - if self.criterion!=None and avg_reward>=self.criterion: # if criterion is not None and average reward is greater than or equal to criterion - return True # return True, meaning that training should end - return False # otherwise, return False, meaning that training should continue - - - def opt(self,state_batch,action_batch,next_state_batch,reward_batch,done_batch,p): - loss=self.nn.loss(state_batch,action_batch,next_state_batch,reward_batch,done_batch,p) # calculate the loss from batch data and process index - if self.priority_flag==True and self.priority_p.value!=-1: # if priority flag is True and priority process index is not -1 - while True: - if self.stop_flag.value==True: # if stop flag is True - return None # return None, meaning that optimization should stop - if p==self.priority_p.value: # if process index is equal to priority process index - break # break the loop and continue optimization - else: - continue # otherwise, continue the loop and wait for priority - if self.stop_func_(): # if stop function returns True - return None # return None, meaning that optimization should stop - loss=loss.clone() # clone the loss tensor - self.nn.backward(loss,p) # perform backward propagation with loss and process index - self.nn.opt(p) # perform optimization with process index - return loss - - - def _train(self,p,j,batches,length): - if j==batches-1: # if it is the last batch - index1=batches*self.batch # get the start index of the batch - index2=self.batch-(length-batches*self.batch) # get the end index of the batch - state_batch=np.concatenate((self.state_pool[p][index1:length],self.state_pool[p][:index2]),0) # get the state batch by concatenating the last part and the first part of state pool - action_batch=np.concatenate((self.action_pool[p][index1:length],self.action_pool[p][:index2]),0) # get the action batch by concatenating the last part and the first part of action pool - next_state_batch=np.concatenate((self.next_state_pool[p][index1:length],self.next_state_pool[p][:index2]),0) # get the next state batch by concatenating the last part and the first part of next state pool - reward_batch=np.concatenate((self.reward_pool[p][index1:length],self.reward_pool[p][:index2]),0) # get the reward batch by concatenating the last part and the first part of reward pool - done_batch=np.concatenate((self.done_pool[p][index1:length],self.done_pool[p][:index2]),0) # get the done batch by concatenating the last part and the first part of done pool - loss=self.opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch,p) # get the loss from opt method with batch data and process index - self.loss[p]+=loss # add loss to loss array for process index - if hasattr(self.nn,'bc'): - bc=self.nn.bc[0] - bc.assign_add(1) - self.nn.bc[0]=bc - else: - index1=j*self.batch - index2=(j+1)*self.batch - state_batch=self.state_pool[p][index1:index2] - action_batch=self.action_pool[p][index1:index2] - next_state_batch=self.next_state_pool[p][index1:index2] - reward_batch=self.reward_pool[p][index1:index2] - done_batch=self.done_pool[p][index1:index2] - loss=self.opt(state_batch,action_batch,next_state_batch,reward_batch,done_batch,p) - self.loss[p]+=loss - if hasattr(self.nn,'bc'): - bc=self.nn.bc[0] - bc.assign_add(1) - self.nn.bc[0]=bc - return - - - def train_(self,p): - if len(self.done_pool[p])=self.max_opt: # if max opt is not None and opt counter for priority process index is greater than or equal to max opt - self.priority_p.value=int(self.priority_p.value) # convert priority process index to integer - elif self.max_opt==None: # if max opt is None - self.priority_p.value=int(self.priority_p.value) # convert priority process index to integer - else: - self.priority_p.value=-1 # set priority process index to -1, meaning no priority - if self.priority_flag==True: - self.opt_counter[p]=0 # reset opt counter for process index to zero - if hasattr(self.nn,'attenuate'): - opt_counter=self.nn.opt_counter[0] - opt_counter[p]=0 - self.nn.opt_counter[0]=opt_counter - self._train(p,j,batches,length) # call _train method with process index, batch index, number of batches and length of done pool - if self.priority_flag==True: - opt_counter=np.frombuffer(self.opt_counter.get_obj(),dtype='i') # get the opt counter array from shared memory - opt_counter+=1 # increment the opt counter array by one - if hasattr(self.nn,'attenuate'): - opt_counter=self.nn.opt_counter[0] - opt_counter+=1 - self.nn.opt_counter[0]=opt_counter - if self.update_step!=None: # if update step is not None - if self.sc[p]%self.update_step==0: # if step counter for process index is divisible by update step - self.nn.update_param() # update the target network parameters - else: - self.nn.update_param() # update the target network parameters - self.loss[p]=self.loss[p]/batches # calculate the average loss for process index by dividing by number of batches - self.sc[p]+=1 # increment the step counter for process index by one - if hasattr(self.nn,'ec'): - ec=self.nn.ec[0] - ec.assign_add(1) - self.nn.ec[0]=ec - return - - - def train(self,p,lock,pool_lock): - lock[1].acquire() # acquire lock for running flag list - self.state_pool[p]=None # initialize state pool for process index to None - self.action_pool[p]=None # initialize action pool for process index to None - self.next_state_pool[p]=None # initialize next state pool for process index to None - self.reward_pool[p]=None # initialize reward pool for process index to None - self.done_pool[p]=None # initialize done pool for process index to None - self.running_flag.append(1) # append one to running flag list, meaning that process is running - self.process_counter.value+=1 # increment the process counter by one - self.finish_list.append(None) # append None to finish list, meaning that process is not finished - lock[1].release() # release lock for running flag list - try: - epsilon=self.epsilon[p] # get the epsilon value for process index - except Exception: - epsilon=None # set epsilon value to None if there is an exception - while True: - if self.stop_flag.value==True: # if stop flag is True - break # break the loop and stop training - if self.episode!=None and self.episode_counter.value>=self.episode: # if episode limit is not None and episode counter is greater than or equal to episode limit - break # break the loop and stop training - s=self.nn.env(p=p,initial=True) # get the initial state from environment with process index - s=np.array(s) - if self.episode_step==None: # if episode step limit is None - while True: - if self.episode!=None and self.episode_counter.value>=self.episode: # if episode limit is not None and episode counter is greater than or equal to episode limit - break # break the loop and stop training - next_s,r,done,index=self.env(s,epsilon,p,lock,pool_lock) # get the next state, reward, done and index from env method with state, epsilon, process index, lock and pool lock - self.reward[p]+=r # add reward to reward array for process index - s=next_s # assign next state to state - if type(self.done_pool[p])==np.ndarray: # if done pool is not empty - self.train_(p) # call train_ method with process index - if self.stop_flag.value==True: # if stop flag is True - break # break the loop and stop training - if done: # if done is True, meaning that episode is finished - if len(lock)==4: # if there are four locks - lock[3].acquire() # acquire the fourth lock for episode counter - self.episode_counter.value+=1 # increment the episode counter by one - self.total_episode.value+=1 # increment the total episode by one - self.loss_list.append(self.loss[p]) # append the loss for process index to loss list - if len(lock)==4: # if there are four locks - lock[3].release() # release the fourth lock for episode counter - break # break the loop and start a new episode - else: # if episode step limit is not None - for l in range(self.episode_step): # loop over episode step limit - if self.episode!=None and self.episode_counter.value>=self.episode: # if episode limit is not None and episode counter is greater than or equal to episode limit - break # break the loop and stop training - next_s,r,done,index=self.env(s,epsilon,p,lock,pool_lock) # get the next state, reward, done and index from env method with state, epsilon, process index, lock and pool lock - self.reward[p]+=r # add reward to reward array for process index - s=next_s # assign next state to state - if type(self.done_pool[p])==np.ndarray: # if done pool is not empty - self.train_(p) # call train_ method with process index - if self.stop_flag.value==True: # if stop flag is True - break # break the loop and stop training - if done: # if done is True, meaning that episode is finished - if len(lock)==4: # if there are four locks - lock[3].acquire() # acquire the fourth lock for episode counter - self.episode_counter.value+=1 # increment the episode counter by one - self.total_episode.value+=1 # increment the total episode by one - self.loss_list.append(self.loss[p]) # append the loss for process index to loss list - if len(lock)==4: # if there are four locks - lock[3].release() # release the fourth lock for episode counter - break # break the loop and start a new episode - if l==self.episode_step-1: # if it is the last step of episode - if len(lock)==4: # if there are four locks - lock[3].acquire() # acquire the fourth lock for episode counter - self.episode_counter.value+=1 - self.total_episode.value+=1 # increment the total episode by one - self.loss_list.append(self.loss[p]) # append the loss for process index to loss list - if len(lock)==4: # if there are four locks - lock[3].release() # release the fourth lock for episode counter - if len(lock)==3 or len(lock)==4: # if there are three or four locks - lock[2].acquire() # acquire the third lock for saving model - self.save_() # call save_ method to save the model - self.reward_list.append(self.reward[p]) # append the reward for process index to reward list - self.reward[p]=0 # reset the reward for process index to zero - if len(lock)==3 or len(lock)==4: # if there are three or four locks - lock[2].release() # release the third lock for saving model - self.running_flag[p+1]=0 # set the running flag for process index to zero, meaning that process is not running - if p not in self.finish_list: # if process index is not in finish list - self.finish_list[p]=p # add process index to finish list, meaning that process is finished - lock[1].acquire() # acquire lock for running flag list - self.process_counter.value-=1 # decrement the process counter by one - lock[1].release() # release lock for running flag list - del self.state_pool[p] # delete state pool for process index - del self.action_pool[p] # delete action pool for process index - del self.next_state_pool[p] # delete next state pool for process index - del self.reward_pool[p] # delete reward pool for process index - del self.done_pool[p] # delete done pool for process index - return - - - def train_online(self,p,lock=None,g_lock=None): - if hasattr(self.nn,'counter'): - self.nn.counter.append(0) - while True: - if hasattr(self.nn,'save'): - self.nn.save(self.save,p) - if hasattr(self.nn,'stop_flag'): - if self.nn.stop_flag==True: - return - if hasattr(self.nn,'stop_func'): - if self.nn.stop_func(p): - return - if hasattr(self.nn,'suspend_func'): - self.nn.suspend_func(p) - try: - data=self.nn.online(p) # get the online data from neural network with process index - except Exception as e: - self.nn.exception_list[p]=e # store the exception in exception list for process index - if data=='stop': # if data is 'stop', meaning that online training should stop - return - elif data=='suspend': # if data is 'suspend', meaning that online training should suspend - self.nn.suspend_func(p) # call suspend function with process index - try: - loss,param=self.opt(data[0],data[1],data[2],data[3],data[4],p,lock,g_lock) # get the loss and parameter from opt method with data, process index, lock and global lock - except Exception as e: - self.nn.exception_list[p]=e # store the exception in exception list for process index - loss=loss.numpy() - if len(self.nn.train_loss_list)==self.nn.max_length: # if train loss list has reached the maximum length - del self.nn.train_loss_list[0] # delete the oldest element from train loss list - self.nn.train_loss_list.append(loss) # append the loss to train loss list - try: - if hasattr(self.nn,'counter'): - count=self.nn.counter[p] - count+=1 - self.nn.counter[p]=count - except IndexError: - self.nn.counter.append(0) - count=self.nn.counter[p] - count+=1 - self.nn.counter[p]=count - return diff --git a/Note 7.0 documentation/RL/neural network/pytorch/non-parallel/DDPG.py b/Note 7.0 documentation/RL/neural network/pytorch/non-parallel/DDPG.py deleted file mode 100644 index 4b0abd94..00000000 --- a/Note 7.0 documentation/RL/neural network/pytorch/non-parallel/DDPG.py +++ /dev/null @@ -1,101 +0,0 @@ -import torch -import torch.nn.functional as F -import numpy as np -import gym - - -class actor(torch.nn.Module): - def __init__(self,state_dim,hidden_dim,action_dim,action_bound): - super(actor,self).__init__() - self.fc1=torch.nn.Linear(state_dim,hidden_dim) - self.fc2=torch.nn.Linear(hidden_dim,action_dim) - self.action_bound=action_bound - - - def forward(self,x): - x=F.relu(self.fc1(x)) - return torch.tanh(self.fc2(x))*self.action_bound - - -class critic(torch.nn.Module): - def __init__(self,state_dim,hidden_dim,action_dim): - super(critic,self).__init__() - self.fc1=torch.nn.Linear(state_dim+action_dim,hidden_dim) - self.fc2=torch.nn.Linear(hidden_dim,1) - - - def forward(self,x,a): - cat=torch.cat([x,a],dim=1) - x=F.relu(self.fc1(cat)) - return self.fc2(x) - - -class DDPG: - def __init__(self,hidden_dim,sigma,gamma,tau,actor_lr,critic_lr): - if torch.cuda.is_available(): - self.device=torch.device('cuda') - else: - self.device=torch.device('cpu') - self.genv=gym.make('Pendulum-v0') - state_dim=self.genv.observation_space.shape[0] - action_dim=self.genv.action_space.shape[0] - action_bound=self.genv.action_space.high[0] - self.actor=actor(state_dim,hidden_dim,action_dim,action_bound).to(self.device) - self.critic=critic(state_dim,hidden_dim,action_dim).to(self.device) - self.target_actor=actor(state_dim,hidden_dim,action_dim,action_bound).to(self.device) - self.target_critic=critic(state_dim,hidden_dim,action_dim).to(self.device) - self.target_actor.load_state_dict(self.actor.state_dict()) - self.target_critic.load_state_dict(self.critic.state_dict()) - self.sigma=sigma - self.gamma=gamma - self.tau=tau - self.actor_opt=torch.optim.Adam(self.actor.parameters(),lr=actor_lr) - self.critic_opt=torch.optim.Adam(self.critic.parameters(),lr=critic_lr) - - - def noise(self): - return np.random.normal(scale=self.sigma) - - - def env(self,a=None,initial=None): - if initial==True: - state=self.genv.reset(seed=0) - return state - else: - next_state,reward,done,_=self.genv.step(a) - return next_state,reward,done - - - def loss(self,s,a,next_s,r,d): - s=torch.tensor(s,dtype=torch.float).to(self.device) - a=torch.tensor(a,dtype=torch.float).view(-1,1).to(self.device) - next_s=torch.tensor(next_s,dtype=torch.float).to(self.device) - r=torch.tensor(r,dtype=torch.float).view(-1,1).to(self.device) - d=torch.tensor(d,dtype=torch.float).view(-1,1).to(self.device) - next_q_value=self.target_critic(next_s,self.target_actor(next_s)) - q_target=r+self.gamma*next_q_value*(1-d) - actor_loss=-torch.mean(self.critic(s,self.actor(s))) - critic_loss=F.mse_loss(self.critic(s,a),q_target) - return [actor_loss,critic_loss] - - - def backward(self,loss): - self.actor_opt.zero_grad() - loss[0].backward() - self.critic_opt.zero_grad() - loss[1].backward() - return - - - def opt(self): - self.actor_opt.step() - self.critic_opt.step() - return - - - def update_param(self): - for target_param,param in zip(self.target_actor.parameters(),self.actor.parameters()): - target_param.data.copy_(target_param.data*(1.0-self.tau)+param.data*self.tau) - for target_param,param in zip(self.target_critic.parameters(),self.critic.parameters()): - target_param.data.copy_(target_param.data*(1.0-self.tau)+param.data*self.tau) - return \ No newline at end of file diff --git a/Note 7.0 documentation/RL/neural network/pytorch/non-parallel/DQN.py b/Note 7.0 documentation/RL/neural network/pytorch/non-parallel/DQN.py deleted file mode 100644 index a0e58689..00000000 --- a/Note 7.0 documentation/RL/neural network/pytorch/non-parallel/DQN.py +++ /dev/null @@ -1,64 +0,0 @@ -import torch -import gym -import torch.nn.functional as F - - -class Qnet(torch.nn.Module): - def __init__(self,state_dim,hidden_dim,action_dim): - super(Qnet,self).__init__() - self.fc1=torch.nn.Linear(state_dim,hidden_dim) - self.fc2=torch.nn.Linear(hidden_dim,action_dim) - - - def forward(self,x): - x=F.relu(self.fc1(x)) - return self.fc2(x) - - -class DQN: - def __init__(self,state_dim,hidden_dim,action_dim): - if torch.cuda.is_available(): - self.device=torch.device('cuda') - else: - self.device=torch.device('cpu') - self.nn=Qnet(state_dim,hidden_dim,action_dim).to(self.device) - self.target_q_net=Qnet(state_dim,hidden_dim,action_dim).to(self.device) - self.optimizer=torch.optim.Adam(self.nn.parameters(),lr=2e-3) #optimizer,kernel uses it to optimize. - self.genv=gym.make('CartPole-v0') #create environment - - - def env(self,a=None,initial=None): #environment function,kernel uses it to interact with the environment. - if initial==True: - state=self.genv.reset() - return state - else: - next_state,reward,done,_=self.genv.step(a) - return next_state,reward,done - - - def loss(self,s,a,next_s,r,d): #loss function,kernel uses it to calculate loss. - s=torch.tensor(s,dtype=torch.float).to(self.device) - a=torch.tensor(a,dtype=torch.int64).view(-1,1).to(self.device) - next_s=torch.tensor(next_s,dtype=torch.float).to(self.device) - r=torch.tensor(r,dtype=torch.float).view(-1,1).to(self.device) - d=torch.tensor(d,dtype=torch.float).view(-1,1).to(self.device) - q_value=self.nn(s).gather(1,a) - next_q_value=self.target_q_net(next_s).max(1)[0].view(-1,1) - target=r+0.98*next_q_value*(1-d) - return F.mse_loss(q_value,target) - - - def backward(self,loss): #backward function,kernel uses it for backpropagation. - self.optimizer.zero_grad() - loss.backward() - return - - - def opt(self): #opt function,kernel uses it to optimize. - self.optimizer.step() - return - - - def update_param(self): #update function,kernel uses it to update parameter. - self.target_q_net.load_state_dict(self.nn.state_dict()) - return \ No newline at end of file diff --git a/Note 7.0 documentation/RL/neural network/pytorch/non-parallel/DQN_pr.py b/Note 7.0 documentation/RL/neural network/pytorch/non-parallel/DQN_pr.py deleted file mode 100644 index 66c71847..00000000 --- a/Note 7.0 documentation/RL/neural network/pytorch/non-parallel/DQN_pr.py +++ /dev/null @@ -1,76 +0,0 @@ -import torch -import gym -import torch.nn.functional as F -import Note.create.RL.rl.prioritized_replay as pr - -#prioritized replay example -class Qnet(torch.nn.Module): - def __init__(self,state_dim,hidden_dim,action_dim): - super(Qnet,self).__init__() - self.fc1=torch.nn.Linear(state_dim,hidden_dim) - self.fc2=torch.nn.Linear(hidden_dim,action_dim) - - - def forward(self,x): - x=F.relu(self.fc1(x)) - return self.fc2(x) - - -class DQN: - def __init__(self,state_dim,hidden_dim,action_dim): - if torch.cuda.is_available(): - self.device=torch.device('cuda') - else: - self.device=torch.device('cpu') - self.nn=Qnet(state_dim,hidden_dim,action_dim).to(self.device) - self.target_q_net=Qnet(state_dim,hidden_dim,action_dim).to(self.device) - self.pr=pr.pr() - self.initial_TD=7 - self._epsilon=0.0007 - self.alpha=0.7 - self.optimizer=torch.optim.Adam(self.nn.parameters(),lr=2e-3) - self.genv=gym.make('CartPole-v0') - - - def env(self,a=None,initial=None): - if initial==True: - state=self.genv.reset(seed=0) - return state - else: - next_state,reward,done,_=self.genv.step(a) - return next_state,reward,done - - - def data_func(self,state_pool,action_pool,next_state_pool,reward_pool,done_pool,batch): - s,a,next_s,r,d=self.pr.sample(state_pool,action_pool,next_state_pool,reward_pool,done_pool,self.epsilon,self.alpha,batch) - return s,a,next_s,r,d - - - def loss(self,s,a,next_s,r,d): - s=torch.tensor(s,dtype=torch.float).to(self.device) - a=torch.tensor(a,dtype=torch.int64).view(-1,1).to(self.device) - next_s=torch.tensor(next_s,dtype=torch.float).to(self.device) - r=torch.tensor(r,dtype=torch.float).view(-1,1).to(self.device) - d=torch.tensor(d,dtype=torch.float).view(-1,1).to(self.device) - q_value=self.nn(s).gather(1,a) - next_q_value=self.target_q_net(next_s).max(1)[0].view(-1,1) - target=r+0.98*next_q_value*(1-d) - TD=target-q_value - self.pr.update_TD(TD) - return torch.mean(TD**2) - - - def backward(self,loss): - self.optimizer.zero_grad() - loss.backward() - return - - - def opt(self): - self.optimizer.step() - return - - - def update_param(self): - self.target_q_net.load_state_dict(self.nn.state_dict()) - return \ No newline at end of file diff --git a/Note 7.0 documentation/RL/neural network/pytorch/non-parallel/DoubleDQN.py b/Note 7.0 documentation/RL/neural network/pytorch/non-parallel/DoubleDQN.py deleted file mode 100644 index 31e606e8..00000000 --- a/Note 7.0 documentation/RL/neural network/pytorch/non-parallel/DoubleDQN.py +++ /dev/null @@ -1,65 +0,0 @@ -import torch -import gym -import torch.nn.functional as F - - -class Qnet(torch.nn.Module): - def __init__(self,state_dim,hidden_dim,action_dim): - super(Qnet,self).__init__() - self.fc1=torch.nn.Linear(state_dim,hidden_dim) - self.fc2=torch.nn.Linear(hidden_dim,action_dim) - - - def forward(self,x): - x=F.relu(self.fc1(x)) - return self.fc2(x) - - -class DoubleDQN: - def __init__(self,state_dim,hidden_dim,action_dim): - if torch.cuda.is_available(): - self.device=torch.device('cuda') - else: - self.device=torch.device('cpu') - self.nn=Qnet(state_dim,hidden_dim,action_dim).to(self.device) - self.target_q_net=Qnet(state_dim,hidden_dim,action_dim).to(self.device) - self.optimizer=torch.optim.Adam(self.nn.parameters(),lr=2e-3) - self.genv=gym.make('CartPole-v0') - - - def env(self,a=None,initial=None): - if initial==True: - state=self.genv.reset(seed=0) - return state - else: - next_state,reward,done,_=self.genv.step(a) - return next_state,reward,done - - - def loss(self,s,a,next_s,r,d): - s=torch.tensor(s,dtype=torch.float).to(self.device) - a=torch.tensor(a,dtype=torch.int64).view(-1,1).to(self.device) - next_s=torch.tensor(next_s,dtype=torch.float).to(self.device) - r=torch.tensor(r,dtype=torch.float).view(-1,1).to(self.device) - d=torch.tensor(d,dtype=torch.float).view(-1,1).to(self.device) - q_value=self.nn(s).gather(1,a) - max_action=self.nn(next_s).max(1)[1].view(-1,1) - next_q_value=self.target_q_net(next_s).gather(1,max_action) - target=r+0.98*next_q_value*(1-d) - return F.mse_loss(q_value,target) - - - def backward(self,loss): - self.optimizer.zero_grad() - loss.backward() - return - - - def opt(self): - self.optimizer.step() - return - - - def update_param(self): - self.target_q_net.load_state_dict(self.nn.state_dict()) - return \ No newline at end of file diff --git a/Note 7.0 documentation/RL/neural network/pytorch/non-parallel/DuelingDQN.py b/Note 7.0 documentation/RL/neural network/pytorch/non-parallel/DuelingDQN.py deleted file mode 100644 index b81eeb37..00000000 --- a/Note 7.0 documentation/RL/neural network/pytorch/non-parallel/DuelingDQN.py +++ /dev/null @@ -1,67 +0,0 @@ -import torch -import gym -import torch.nn.functional as F - - -class VAnet(torch.nn.Module): - def __init__(self,state_dim,hidden_dim,action_dim): - super(VAnet,self).__init__() - self.fc1=torch.nn.Linear(state_dim,hidden_dim) - self.fc_A=torch.nn.Linear(hidden_dim,action_dim) - self.fc_V=torch.nn.Linear(hidden_dim,1) - - - def forward(self,x): - A=self.fc_A(F.relu(self.fc1(x))) - V=self.fc_V(F.relu(self.fc1(x))) - Q=V+A-A.mean(1).view(-1,1) - return Q - - -class DuelingDQN: - def __init__(self,state_dim,hidden_dim,action_dim): - if torch.cuda.is_available(): - self.device=torch.device('cuda') - else: - self.device=torch.device('cpu') - self.nn=VAnet(state_dim,hidden_dim,action_dim).to(self.device) - self.target_q_net=VAnet(state_dim,hidden_dim,action_dim).to(self.device) - self.optimizer=torch.optim.Adam(self.nn.parameters(),lr=2e-3) - self.genv=gym.make('CartPole-v0') - - - def env(self,a=None,initial=None): - if initial==True: - state=self.genv.reset(seed=0) - return state - else: - next_state,reward,done,_=self.genv.step(a) - return next_state,reward,done - - - def loss(self,s,a,next_s,r,d): - s=torch.tensor(s,dtype=torch.float).to(self.device) - a=torch.tensor(a,dtype=torch.int64).view(-1,1).to(self.device) - next_s=torch.tensor(next_s,dtype=torch.float).to(self.device) - r=torch.tensor(r,dtype=torch.float).view(-1,1).to(self.device) - d=torch.tensor(d,dtype=torch.float).view(-1,1).to(self.device) - q_value=self.nn(s).gather(1,a) - next_q_value=self.target_q_net(next_s).max(1)[0].view(-1,1) - target=r+0.98*next_q_value*(1-d) - return F.mse_loss(q_value,target) - - - def backward(self,loss): - self.optimizer.zero_grad() - loss.backward() - return - - - def opt(self): - self.optimizer.step() - return - - - def update_param(self): - self.target_q_net.load_state_dict(self.nn.state_dict()) - return \ No newline at end of file diff --git a/Note 7.0 documentation/RL/neural network/pytorch/parallel/DQN.py b/Note 7.0 documentation/RL/neural network/pytorch/parallel/DQN.py deleted file mode 100644 index f4d8417d..00000000 --- a/Note 7.0 documentation/RL/neural network/pytorch/parallel/DQN.py +++ /dev/null @@ -1,65 +0,0 @@ -import torch -import gym -import torch.nn.functional as F -from Note.nn.parallel.assign_device_pytorch import assign_device - - -class Qnet(torch.nn.Module): - def __init__(self,state_dim,hidden_dim,action_dim): - super(Qnet,self).__init__() - self.fc1=torch.nn.Linear(state_dim,hidden_dim) - self.fc2=torch.nn.Linear(hidden_dim,action_dim) - - - def forward(self,x): - x=F.relu(self.fc1(x)) - return self.fc2(x) - - -class DQN: - def __init__(self,state_dim,hidden_dim,action_dim): - if torch.cuda.is_available(): - self.device=torch.device('cuda') - else: - self.device=torch.device('cpu') - self.nn=Qnet(state_dim,hidden_dim,action_dim).to(self.device) - self.target_q_net=Qnet(state_dim,hidden_dim,action_dim).to(self.device) - self.optimizer=[torch.optim.Adam(self.nn.parameters(),lr=2e-3) for _ in range(5)] #optimizer,kernel uses it to optimize. - self.genv=[gym.make('CartPole-v0') for _ in range(5)] #create environment - - - def env(self,a=None,p=None,initial=None): #environment function,kernel uses it to interact with the environment. - if initial==True: - state=self.genv[p].reset() - return state - else: - next_state,reward,done,_=self.genv[p].step(a) - return next_state,reward,done - - - def loss(self,s,a,next_s,r,d,p): #loss function,kernel uses it to calculate loss. - s=torch.tensor(s,dtype=torch.float).to(assign_device(p,'GPU')) - a=torch.tensor(a,dtype=torch.int64).view(-1,1).to(assign_device(p,'GPU')) - next_s=torch.tensor(next_s,dtype=torch.float).to(assign_device(p,'GPU')) - r=torch.tensor(r,dtype=torch.float).view(-1,1).to(assign_device(p,'GPU')) - d=torch.tensor(d,dtype=torch.float).view(-1,1).to(assign_device(p,'GPU')) - q_value=self.nn(s).gather(1,a) - next_q_value=self.target_q_net(next_s).max(1)[0].view(-1,1) - target=r+0.98*next_q_value*(1-d) - return F.mse_loss(q_value,target) - - - def backward(self,loss,p): #backward function,kernel uses it for backpropagation. - self.optimizer[p].zero_grad(set_to_none=True) - loss.backward() - return - - - def opt(self,p): #opt function,kernel uses it to optimize. - self.optimizer[p].step() - return - - - def update_param(self): #update function,kernel uses it to update parameter. - self.target_q_net.load_state_dict(self.nn.state_dict()) - return \ No newline at end of file diff --git a/Note 7.0 documentation/RL/neural network/tensorflow/non-parallel/DDPG.py b/Note 7.0 documentation/RL/neural network/tensorflow/non-parallel/DDPG.py deleted file mode 100644 index d3666ee1..00000000 --- a/Note 7.0 documentation/RL/neural network/tensorflow/non-parallel/DDPG.py +++ /dev/null @@ -1,84 +0,0 @@ -import tensorflow as tf # import TensorFlow library -import numpy as np # import NumPy library -import gym # import OpenAI Gym library - - -class actor: # define a class for the actor network - def __init__(self,state_dim,hidden_dim,action_dim,action_bound): # initialize the network with state dimension, hidden dimension, action dimension and action bound - self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) # create a weight matrix for the first layer - self.bias1=tf.Variable(tf.random.normal([hidden_dim])) # create a bias vector for the first layer - self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) # create a weight matrix for the second layer - self.bias2=tf.Variable(tf.random.normal([action_dim])) # create a bias vector for the second layer - self.action_bound=action_bound # store the action bound - self.param=[self.weight1,self.bias1,self.weight2,self.bias2] # store the network parameters in a list - - - def fp(self,x): # forward propagation function, kernel uses it for forward propagation - x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) # apply the first layer with ReLU activation - return tf.nn.tanh(tf.matmul(x,self.weight2)+self.bias2)*self.action_bound # apply the second layer with tanh activation and scale it by the action bound - - -class critic: # define a class for the critic network - def __init__(self,state_dim,hidden_dim,action_dim): # initialize the network with state dimension, hidden dimension and action dimension - self.weight1=tf.Variable(tf.random.normal([state_dim+action_dim,hidden_dim])) # create a weight matrix for the first layer - self.bias1=tf.Variable(tf.random.normal([hidden_dim])) # create a bias vector for the first layer - self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) # create a weight matrix for the second layer - self.bias2=tf.Variable(tf.random.normal([action_dim])) # create a bias vector for the second layer - self.param=[self.weight1,self.bias1,self.weight2,self.bias2] # store the network parameters in a list - - - def fp(self,x,a): # forward propagation function, kernel uses it for forward propagation - cat=tf.concat([x,a],axis=1) # concatenate the state and action vectors along the first axis - x=tf.nn.relu(tf.matmul(cat,self.weight1)+self.bias1) # apply the first layer with ReLU activation - return tf.matmul(x,self.weight2)+self.bias2 # apply the second layer and return the output - - -class DDPG: # define a class for the DDPG agent - def __init__(self,hidden_dim,sigma,gamma,tau,actor_lr,critic_lr): # initialize the agent with hidden dimension, noise scale, discount factor, soft update factor, actor learning rate and critic learning rate - self.genv=gym.make('Pendulum-v0') # create environment - state_dim=self.genv.observation_space.shape[0] # get the state dimension from the environment observation space - action_dim=self.genv.action_space.shape[0] # get the action dimension from the environment action space - action_bound=self.genv.action_space.high[0] # get the action bound from the environment action space high value (assume symmetric) - self.actor=actor(state_dim,hidden_dim,action_dim,action_bound) # create an actor network for the agent - self.critic=critic(state_dim,hidden_dim,action_dim) # create a critic network for the agent - self.target_actor=actor(state_dim,hidden_dim,action_dim,action_bound) # create a target actor network for the agent - self.target_critic=critic(state_dim,hidden_dim,action_dim) # create a target critic network for the agent - self.actor_param=self.actor.param # parameter list of actor network, kernel uses it list for backpropagation - self.critic_param=self.critic.param # parameter list of critic network, kernel uses it list for backpropagation - self.target_actor.param=self.actor_param.copy() # copy the parameters from actor network to target actor network - self.target_critic.param=self.critic_param.copy() # copy the parameters from critic network to target critic network - self.param=[self.actor_param,self.critic_param] # parameter list of both networks, kernel uses it list for backpropagation - self.sigma=sigma # noise scale - self.gamma=gamma # discount factor - self.tau=tau # soft update factor - self.opt=tf.keras.optimizers.Adam() # optimizer, kernel uses it to optimize. Here we use Adam optimizer - - - def noise(self): # noise function, kernel uses it to generate exploration noise - return np.random.normal(scale=self.sigma) # return a random sample from a normal distribution with zero mean and sigma scale - - - def env(self,a=None,initial=None): # environment function, kernel uses it to interact with the environment - if initial==True: # if initial is True, reset the environment and return the initial state - state=self.genv.reset(seed=0) - return state - else: # otherwise, take an action and return the next state, reward and done flag - next_state,reward,done,_=self.genv.step(a) - return next_state,reward,done - - - def loss(self,s,a,next_s,r,d): # loss function, kernel uses it to calculate loss - a=tf.expand_dims(a,axis=1) # expand the action vector to match the Q-value matrix shape - next_q_value=self.target_critic.fp(next_s,self.target_actor.fp(next_s)) # get the Q-value for the next state and action from the target critic network - q_target=r+self.gamma*next_q_value*(1-d) # calculate the target value using Bellman equation with discount factor gamma - actor_loss=-tf.reduce_mean(self.critic.fp(s,self.actor.fp(s))) # calculate the actor loss as the negative mean of the Q-value for the current state and action from the critic network - critic_loss=tf.reduce_mean((self.critic.fp(s,a)-q_target)**2) # calculate the critic loss as the mean squared error between Q-value and target value - return [actor_loss,critic_loss] # return a list of actor loss and critic loss - - - def update_param(self): # update function, kernel uses it to update parameter - for target_param,param in zip(self.target_actor.param,self.actor.param): # for each pair of parameters in target actor network and actor network - target_param=target_param*(1.0-self.tau)+param*self.tau # update the target parameter using soft update with factor tau - for target_param,param in zip(self.target_critic.param,self.critic.param): # for each pair of parameters in target critic network and critic network - target_param=target_param*(1.0-self.tau)+param*self.tau # update the target parameter using soft update with factor tau - return \ No newline at end of file diff --git a/Note 7.0 documentation/RL/neural network/tensorflow/non-parallel/DQN.py b/Note 7.0 documentation/RL/neural network/tensorflow/non-parallel/DQN.py deleted file mode 100644 index 21487674..00000000 --- a/Note 7.0 documentation/RL/neural network/tensorflow/non-parallel/DQN.py +++ /dev/null @@ -1,46 +0,0 @@ -import tensorflow as tf # import TensorFlow library -import gym # import OpenAI Gym library - - -class Qnet: # define a class for the Q-network - def __init__(self,state_dim,hidden_dim,action_dim): # initialize the network with state dimension, hidden dimension and action dimension - self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) # create a weight matrix for the first layer - self.bias1=tf.Variable(tf.random.normal([hidden_dim])) # create a bias vector for the first layer - self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) # create a weight matrix for the second layer - self.bias2=tf.Variable(tf.random.normal([action_dim])) # create a bias vector for the second layer - self.param=[self.weight1,self.bias1,self.weight2,self.bias2] # store the network parameters in a list - - def fp(self,x): # forward propagation function, kernel uses it for forward propagation - x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) # apply the first layer with ReLU activation - return tf.matmul(x,self.weight2)+self.bias2 # apply the second layer and return the output - - -class DQN: # define a class for the DQN agent - def __init__(self,state_dim,hidden_dim,action_dim): # initialize the agent with state dimension, hidden dimension and action dimension - self.nn=Qnet(state_dim,hidden_dim,action_dim) # create a Q-network for the agent - self.target_q_net=Qnet(state_dim,hidden_dim,action_dim) # create a target Q-network for the agent - self.param=self.nn.param # parameter list, kernel uses it list for backpropagation - self.opt=tf.keras.optimizers.Adam() # optimizer, kernel uses it to optimize - self.genv=gym.make('CartPole-v0') # create environment - - - def env(self,a=None,initial=None): # environment function, kernel uses it to interact with the environment - if initial==True: # if initial is True, reset the environment and return the initial state - state=self.genv.reset(seed=0) - return state - else: # otherwise, take an action and return the next state, reward and done flag - next_state,reward,done,_=self.genv.step(a) - return next_state,reward,done - - - def loss(self,s,a,next_s,r,d): # loss function, kernel uses it to calculate loss - a=tf.expand_dims(a,axis=1) # expand the action vector to match the Q-value matrix shape - q_value=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) # get the Q-value for the selected action - next_q_value=tf.reduce_max(self.target_q_net.fp(next_s),axis=1) # get the maximum Q-value for the next state from the target network - target=r+0.98*next_q_value*(1-d) # calculate the target value using Bellman equation with discount factor 0.98 - return tf.reduce_mean((q_value-target)**2) # return the mean squared error between Q-value and target value - - - def update_param(self): # update function, kernel uses it to update parameter - self.target_q_net.param=self.param.copy() # copy the parameters from the Q-network to the target network - return \ No newline at end of file diff --git a/Note 7.0 documentation/RL/neural network/tensorflow/non-parallel/DQN_pr.py b/Note 7.0 documentation/RL/neural network/tensorflow/non-parallel/DQN_pr.py deleted file mode 100644 index 77e8b8a9..00000000 --- a/Note 7.0 documentation/RL/neural network/tensorflow/non-parallel/DQN_pr.py +++ /dev/null @@ -1,59 +0,0 @@ -import tensorflow as tf -import gym -import Note.create.RL.rl.prioritized_replay as pr - - -class Qnet: - def __init__(self,state_dim,hidden_dim,action_dim): - self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) - self.bias1=tf.Variable(tf.random.normal([hidden_dim])) - self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) - self.bias2=tf.Variable(tf.random.normal([action_dim])) - self.param=[self.weight1,self.bias1,self.weight2,self.bias2] - - def fp(self,x): - x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) - return tf.matmul(x,self.weight2)+self.bias2 - - -class DQN: - def __init__(self,state_dim,hidden_dim,action_dim): - self.nn=Qnet(state_dim,hidden_dim,action_dim) - self.target_q_net=Qnet(state_dim,hidden_dim,action_dim) - self.param=self.nn.param - self.pr=pr.pr() - self.initial_TD=7 - self._epsilon=0.0007 - self.alpha=0.7 - self.opt=tf.keras.optimizers.Adam() - self.genv=gym.make('CartPole-v0') - - - def env(self,a=None,initial=None): - if initial==True: - state=self.genv.reset(seed=0) - return state - else: - next_state,reward,done,_=self.genv.step(a) - return next_state,reward,done - - - def data_func(self,state_pool,action_pool,next_state_pool,reward_pool,done_pool,batch): - s,a,next_s,r,d=self.pr.sample(state_pool,action_pool,next_state_pool,reward_pool,done_pool,self.epsilon,self.alpha,batch) - return s,a,next_s,r,d - - - def loss(self,s,a,next_s,r,d): - a=tf.expand_dims(a,axis=1) - q_value=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) - next_q_value=tf.reduce_max(self.target_q_net.fp(next_s),axis=1) - target=r+0.98*next_q_value*(1-d) - target=tf.expand_dims(target,axis=1) - TD=target-q_value - self.pr.update_TD(TD) - return tf.reduce_mean(TD**2) - - - def update_param(self): - self.target_q_net.param=self.param.copy() - return \ No newline at end of file diff --git a/Note 7.0 documentation/RL/neural network/tensorflow/parallel/DQN.py b/Note 7.0 documentation/RL/neural network/tensorflow/parallel/DQN.py deleted file mode 100644 index d7756475..00000000 --- a/Note 7.0 documentation/RL/neural network/tensorflow/parallel/DQN.py +++ /dev/null @@ -1,52 +0,0 @@ -import tensorflow as tf # import TensorFlow library -import gym # import OpenAI Gym library -import Note.nn.parallel.optimizer as o # import Note's optimizer module - - -class Qnet: # define a class for the Q-network - def __init__(self,state_dim,hidden_dim,action_dim): # initialize the network with state dimension, hidden dimension and action dimension - self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) # create a weight matrix for the first layer - self.bias1=tf.Variable(tf.random.normal([hidden_dim])) # create a bias vector for the first layer - self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) # create a weight matrix for the second layer - self.bias2=tf.Variable(tf.random.normal([action_dim])) # create a bias vector for the second layer - self.param=[self.weight1,self.bias1,self.weight2,self.bias2] # store the network parameters in a list - - def fp(self,x): # forward propagation function, kernel uses it for forward propagation - x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) # apply the first layer with ReLU activation - return tf.matmul(x,self.weight2)+self.bias2 # apply the second layer and return the output - - -class DQN: # define a class for the DQN agent - def __init__(self,state_dim,hidden_dim,action_dim): # initialize the agent with state dimension, hidden dimension and action dimension - self.nn=Qnet(state_dim,hidden_dim,action_dim) # create a Q-network for the agent - self.target_q_net=Qnet(state_dim,hidden_dim,action_dim) # create a target Q-network for the agent - self.param=self.nn.param # parameter list, kernel uses it list for backpropagation - self.optimizer=o.Momentum(0.07,0.7) # optimizer, kernel uses it to optimize. Here we use a custom momentum optimizer with learning rate 0.07 and momentum 0.7 - self.genv=[gym.make('CartPole-v0') for _ in range(5)] # create a list of 5 environments - - - def env(self,a=None,p=None,initial=None): # environment function, kernel uses it to interact with the environment - if initial==True: # if initial is True, reset the environment with index p and return the initial state - state=self.genv[p].reset(seed=0) - return state - else: # otherwise, take an action and return the next state, reward and done flag from the environment with index p - next_state,reward,done,_=self.genv[p].step(a) - return next_state,reward,done - - - def loss(self,s,a,next_s,r,d): # loss function, kernel uses it to calculate loss - a=tf.expand_dims(a,axis=1) # expand the action vector to match the Q-value matrix shape - q_value=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) # get the Q-value for the selected action - next_q_value=tf.reduce_max(self.target_q_net.fp(next_s),axis=1) # get the maximum Q-value for the next state from the target network - target=r+0.98*next_q_value*(1-d) # calculate the target value using Bellman equation with discount factor 0.98 - return tf.reduce_mean((q_value-target)**2) # return the mean squared error between Q-value and target value - - - def update_param(self): # update function, kernel uses it to update parameter - self.target_q_net.param=self.param.copy() # copy the parameters from the Q-network to the target network - return - - - def opt(self,gradient): # optimization function, kernel uses it to optimize parameter - param=self.optimizer.opt(gradient,self.param) # apply the custom momentum optimizer to update the parameters using the gradient - return param \ No newline at end of file diff --git a/Note 7.0 documentation/compiler/nc.txt b/Note 7.0 documentation/compiler/nc.txt deleted file mode 100644 index 14ea9e06..00000000 --- a/Note 7.0 documentation/compiler/nc.txt +++ /dev/null @@ -1,33 +0,0 @@ -Note Compiler - -Note Compiler can compile filename.n into filename.py. -Using Note compiler can use different grammar to write your neural network in filename.n. - -Grammar transition by Note Compiler: -Operator: -(object.*object)->tf.matmul(object1,object2) -(object.^)->tf.reverse(object) -(object1.|object2)->tf.concat([object1,object2],0) -(object1.||object2)->tf.concat([object1,object2],1) -(object./)->tf.math.sqrt(object) -(object1.=object2)->state_ops.assign(object1,object2) - -Init: -example: -param=zint32. [3,4]->param=tf.zeros([3,4],dtype=tf.int32) -return zint32. [3,4]->return tf.zeros([3,4],dtype=tf.int32) -param=uint32. (0,1)[3,4]->param=tf.random.uniform([3,4],0,1,dtype=tf.int32) -return uint32. (0,1)[3,4]->return tf.random.uniform([3,4],0,1,dtype=tf.int32) -param=nint32. (0,1)[3,4]->param=tf.random.normal([3,4],0,1,dtype=tf.int32) -return nint32. (0,1)[3,4]->return tf.random.normal([3,4],0,1,dtype=tf.int32) -param=oint32. [3,4]->param=tf.ones([3,4],dtype=tf.int32) -return oint32. [3,4]->return tf.ones([3,4],dtype=tf.int32) - -Using example: -import Note.create.nc as nc -c=nc.compiler('nn.n') -c.Compile() -#then you will get filename.py. - -Macro: -define. macro name string \ No newline at end of file diff --git a/Note 7.0 documentation/compiler/nn.n b/Note 7.0 documentation/compiler/nn.n deleted file mode 100644 index fdf619f1..00000000 --- a/Note 7.0 documentation/compiler/nn.n +++ /dev/null @@ -1,26 +0,0 @@ -import tensorflow as tf -define. Pi 3.14 - -class nn: #a simple example,a neural network class - def __init__(self): - self.weight1=tf.Variable(nfloat32. [784,64]) - self.bias1=tf.Variable(nfloat32. [64]) - self.weight2=tf.Variable(nfloat32. [64,64]) - self.bias2=tf.Variable(nfloat32. [64]) - self.weight3=tf.Variable(nfloat32. [64,10]) - self.bias3=tf.Variable(nfloat32. [10]) - self.param=[self.weight1,self.weight2,self.weight3,self.bias1,self.bias2,self.bias3] - self.opt=tf.keras.optimizers.Adam() - self.info='example' - - - def fp(self,data): - with tf.device('GPU:0'): - layer1=tf.nn.relu((data.*self.weight1)+self.bias1) - layer2=tf.nn.relu((layer1.*self.weight2)+self.bias2) - output=(layer2.*self.weight3)+self.bias3+Pi - return output - - - def loss(self,output,labels): - return tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=output,labels=labels)) \ No newline at end of file diff --git a/Note 7.0 documentation/compiler/nn.py b/Note 7.0 documentation/compiler/nn.py deleted file mode 100644 index 7d02bda4..00000000 --- a/Note 7.0 documentation/compiler/nn.py +++ /dev/null @@ -1,25 +0,0 @@ -import tensorflow as tf - -class nn: #a simple example,a neural network class - def __init__(self): - self.weight1=tf.Variable(tf.random.normal([784,64],dtype=tf.float32)) - self.bias1=tf.Variable(tf.random.normal([64],dtype=tf.float32)) - self.weight2=tf.Variable(tf.random.normal([64,64],dtype=tf.float32)) - self.bias2=tf.Variable(tf.random.normal([64],dtype=tf.float32)) - self.weight3=tf.Variable(tf.random.normal([64,10],dtype=tf.float32)) - self.bias3=tf.Variable(tf.random.normal([10],dtype=tf.float32)) - self.param=[self.weight1,self.weight2,self.weight3,self.bias1,self.bias2,self.bias3] - self.opt=tf.keras.optimizers.Adam() - self.info='example' - - - def fp(self,data): - with tf.device('GPU:0'): - layer1=tf.nn.relu(tf.matmul(data,self.weight1)+self.bias1) - layer2=tf.nn.relu(tf.matmul(layer1,self.weight2)+self.bias2) - output=tf.matmul(layer2,self.weight3)+self.bias3+3.14 - return output - - - def loss(self,output,labels): - return tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=output,labels=labels)) \ No newline at end of file From 7546765508aa4d4e3eb9eb5559c6f28b5f3a1a81 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Wed, 16 Aug 2023 14:06:31 +0800 Subject: [PATCH 173/337] Delete picture directory --- picture/Pool Net.png | Bin 41040 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 picture/Pool Net.png diff --git a/picture/Pool Net.png b/picture/Pool Net.png deleted file mode 100644 index 16c35f8f8e5096685841308e99bfe874281058e1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 41040 zcmeFZZq3eN%-u&(Zb$Vg2m=yBo)qm3z-;h7+7j zTnSW~pt+D|3{QC2f8nY~`zOFYlR_qhpn<oVC5B@*%VbD@VOdYWGIA6LnRnYFb zkn&G{PClHX-Dbd1abL}@;b%F)wJMl6#2|zs@{QQGWjAkW(k&VqV{+C^MmCB<<`+cgX@BA zyZ77>ZvKaLSJ!Z<8n?5ruAA}ioB!5Zws4sYHnz43-=wrZ^jgo?^!l_fxvaH+*08-D zDPO+L(B?U)G~|#*ZQ&q#!fU*_zqYlx=Q?|hU8Y&vCvvmsc-)O6iV0p!D0|xyi|cE* z;(eo;`=?T0)b*ge;jipPw&!uRsJ`#x)$z{-8%GU&?|K&PpH}9UnU~YwZk7+Hk1Jf9 zCA9-_nfoDG7d_-RiwvI46fAu#+_i??$QqS;B7CO9B(pUpLscnsQh7ZGB((Sg?az<4 zdM3)QQRH?t&}O|S+~#nu`ugVi#`k$g z(DnFmdo1T{dm_ zXOk(kn;Uqzvf{o&aevYNbj5Lh;rnz!C4_58AZEZIrKXZgFA*3u5lpPNyZ`4$?bS&? z1dkj(8Sdk|e}xvQJ`ZQV5!#VU)a6`-{g|iizj3}qv90lmOts`xk%=bhb{=yR({I(Qe=&31xTBGw(Gw`6uu8ZoRHAg{ zbR@AUxM)<8?eX8d`*>1BFA8ZvK;}<8`M9NxHsT6``k6F#YpUL)rp7$3;QXO-e%!dW(=H4bYpnzUw2i5rKrPgGQmHxAx+b%6DwRGN zH{AxCCE90>WD0X!v}9HA6(+gm{5W2ry!!PES49GxUcwsLuy?NWCMgKdZMk6exkvfC zkJ_J~7%|z<3ZL-e+OKA`kHKiGc(>nq-_7v^<30ygsF2s~(Qakul}2Ct)YQyNTqmzG zwcWF2J^16_JSy@nZBn{#;w>f8s)GIH-)RA5Cml3T>sRxT7dOKc45|Ee_g~Ke|Ni#^ z1(MMhqzC>lu=zXQev&{nizXP>F2uYBGqvD7B=U)T+2`JRh`dBN5uMg|4@=PH&|H%h zftw~j089yqaU!m<5m8#WLQw-D*yg->yx8qU{+8o?!mh_U1SOy}H@Tcko3l%?OBe)x zAX~ck z6o|O*j^`EW5!aTuy_s6}TJ=ZKx*7SmlViIiBC&};j%|Ks7D7wqRKrR1T_KH&;m+fW z@Awao32EY&ckW0?avk*+9R`=-xGDf%q9WGFW~$*?63G~XeVD|e!OO|3%ddx4Q>ymk zYsnYHQnBp__RMwk-@#!R7(}d9s>Go$>U1xfBp|Ty<=9OB8O5wd3(08)M&pU*W5#!N7%pj^Lb%PcSG`H9jOMW?tj%q z5^+_6#Ed)q;J1q(!BRA*=y?nTJb)_Z?!zW$vPkfo!c|8-=VLBnFp}KW{}-y2DV0AyPcgcC61Ehs_Us_Uk=gk>37bjLHw%2peL?;gXOJt5VLT z_XiD?OP zyw1b10^WgcI=mggdq0{bM0{|nLe6V7XK+KMm7l-;t4LA??H(Ta3o){mc=P$2Uxc)* zur)F`gbz^p>lG*%6 z8gtB=c@2BZAr}lZrjs?{>ILHMTQ@B2FMFRFH<(_1V|hXFeS?h%5gZUJ(V8=b!H@P_ zavgc4?_((kXCjor^a}@zq1ZcvQX6qatx!_vkIw)nc=%=tW9TG|YxSjujv>>1LF~J~ zO5J8T-apYgf+1{fI3jnu1r-1(xKQ)o{rB#;NCR1Gp6`Rl_2HiSGhVJCnlgR_8H7_S zSila{%WvToOi~uGHzAE+<0x8y0akN{(PjvXdSjrU-T%4=kSd3{oA-L5_X4m~zoG|af_!72Z#00V><1iu8 z|C_1KZgO!RwgCpb_~II|qP_98HTek2a#h0Zy6NvK-NIJj+?r}KC_Gbo?B+;WYBh(H*kZF!f?g&#bn+3 zRyzaBxr@I4SWEkWBv&*5fw%DB;zzVrF$7&d3~OirhF7Z+FuT_w zGIW-Tp;ckzzJ23)eW>co#v$Gi^fM?_#Gky_&!{g7%~O+B;>|fq%7J>hmgY8U8yPPd zLtAAlLe85Radixi>PfM^a(WXX1euV}rLq!;;jpJi$>q`oU%JVjNfbZ>1k|0f?Gd71 zdHGTZI@vkyjMmRauGG@3XR|t_0B|-Vv-PGs?u-KZ?TlICgb3*i2JQJeY~p3*Y?X`T zUba*wea44P3iK5MeT^(;4BMbZ!<*J|NRi|Xh4mD`o)?=3?uzp@CL@G#SoWa!xrS2t zG$JXHI7ciRPXG6rcP=f>%Gh)OB68M4(Cg5AO@=H9PW(l3(N(~DMAOt-ws=Xb-Yn@238Ulr!}CBWN^MsQH1blzmBkz^8wpmAy7g+8jJh&SEn_q-G9xlCpMa| ze6BwgE7(8YUDaBtLO-}OPB`~@PGQKUs{|~$EQ~lG&P^PX(gc9x3whc8bXO_>pP0<_ zuq#DJXT^=O! z79ge$C8T`{*SZDUQswI#RVn@MFl>%f7G(ql%7ICR)Hj_Ml}Aj7xlkPY+rG_8=YyGR z779n&JKWubf7FomKy_c819|OYYy@N>f>fi`eIAB0l|h{D@1)4XQ9?8iQeeP%!n?R~ zQ<~eK4K_k$n0sMZU+}d~@CN^vL`9@%R5I(Mzx5-&;knD9hE`6E5gDKKo|r24_YRFkfw&qa0T9?$smZB=Fb$Q4=N}_0Q3e_$@NkG4a~7$ z#Z<3~oZLsB zuO=B~TxuH3zUhnvS_(D9%G=YhRhk#wj~!$1RIJ<8G(CWo1Kmlhy7S*1U{0sQ+&`%0 zShkgR09kQ~eRuc04<2m=)XoSj(lmfR=2MY4&V-Z$HeyAWL)={sW=0w;5z!cv=Fo&M zhPiotBe`(cc4$e=&8fTJUz5|xWk?YAaqZf>p&&jjA{6Z8*adA(*W|G_U7Oq)>XGZEa7;GF?)e^Hhc)g7V z8D{CFX6JHgGB(-$bXh;FjX{>mx1Bff?`}TJrC}(5z08zpH;mT74Le^1QUlbmE2j0+N|RPPH|h0fhnpqW zoD4qO>_;Df9PMZIecQs@r3YwU$es>-&j!b6{$R-JUHgmsTxzoLU~ydJR6SE@uGhcw zpHKY!r;)euQ9?S12-c0}97_Cai2A>Ll#BRQ-$sgJ!V_$eCZKXxdtB|CkO^xt2K7Oi z<_e{*ad8d5MC{QP@n}6VU)^Roqv1P!2DtUQ7c6EgHq;N3(?g`SH0Tr`NsgAqu3dUz zBb1j}S{a;Avn=Q2di(o-BGT}`e#g_jPoKN0pVi}zKW3?H&1cV}rQQhBc*#9?ir;ma z*c5`Pt%MI6c}*`UoEV$wdMV3(^{#u9(wYow0$@ti-(*B!bswl8?;&b>$ouQUjs9v{ zbs8CqImDOA`$A9dG%gO4H4Hge9PA(+iv$=UIlfQxBbfqI%XB>7{-AnV{OSlJ|D`b) zCcgGw294y?WgnSss-7%rC2Ru*jRc4da9FEGyg^{7VRH&!o6EvD{p$isD1u}iZ&({l zi@`v4pjgae7MZTpU-1Uq&}OnwmNzPun)b))O%H@tR3s6-D-w?pNu@-I!WC#F$m$Y= zP5xku&=+o!?z;Ejy}$Lys*)3jE|2r%PW8VWpS8P_u?@VzcdTV^TtztW8su?Xx!Rlh z&0{{^fkpDBqm1rUN)X5--FoSjg~Ew%tTsJy&YNMSoP(5M_1^v8R*I4BYnFV#LscM7 za)xNDuDoddkl+dN9)p$RrIrjazS*GB&!y`1Vs4|E0=zG99`UVDWM4M!nDvC=sjC*t z<2JZ+v9f9Z#Prnk7n>*5uge|L_;i!1?>`B&9kX{|g&TrS5kO^E#+BikBS0~4r%^_) zIZVc;Rqn|h-WrsKXkMPItSAt0{@242$^KsHNj3<(l88+!^%*i@ZZy)r(Q~1cH~TV1&DIZ^X-PJ@wU90jXK0w3OU4p#nrVal=sm zCv+T67@033FW93@(&>KXJGSq5()1|cddT^^ak=5=nfvEwlZh_jo9h?8_Bc7(qHzb- z^4_kGH4RI?rotPPJ{AW4IVv{yiyyjvew!YYQ#N;f2u|8Wov*hFw9*vv>@E6vVj#+_CfdKQ-n`(E8kct*Q(; zm@U;SbkbZmqWS)M{#$e$8_}*Z5TBDjk75IoyyeZDhC9ow_2}JmnQjYjB>-3MK(oBE zHwoqjl3FX^lbtsQU<3@6;(^Gd?HYP5IWwluf5R80Nf}hAq1xtTwqWNQ%noCy6hQMq zxkz^JU|)@;p4cwg5{wFMJo>p%CK-X<2@?;pe4qYZ-|Mcw><9f{(I}We;I0`I z|9+_j+09}f44q(yc@x1EDz|pWmzGU??G(qw?Qd6hl3otSV(c`H_*#j~BHKM`h31?z zyR?ic-d|OX=djWKdI&aWK9mkKuLkPpMNFrB$jn*pEkG331g41?1c<{W^ zM%Q~U`TlIwpPq1Gc7i?3= zwwzSZB}UD|QP1D|#m!dewsceqnT=+)it;XYRBGcEypJ6%RV|WR2I~r}7~?cycL%*% z(~J6Ody`UHLz&W*5I{=ucwdxDJcv+5prq z=%~1PUzItVFCE!Y!ITgxVR6Rv>+xEf+p-1l*%$3DdWfKD;BrmjNrgfoB*;CJrq4jR z$(YMvCI2NQE{UQ&Rjjz=JgaL)TEWr&Fu9o4`<#mz9+plsZn!afo1mZrBtBT%Q}=n9 zPY)1=Z$Ms-S*Cr+0+7oP+FhUs3-UcPmMwIg-7(Uru+bWLnDz5;Gsi;|7et5<5ao9=9b-7UW&fq`WP4VS=H=&Y zq#MfxEP8DqX3O3h{1XX)XurwC;Fb4SADwq=+kUW=M$DGLl z42vLHLuxIVMI|`qh+?oR$w^YFu^kzC-MTF<&ZX+*udilJU5M8`elgbWpTc*^dIOAf zX|Oe5JXd9C0|elBhfJJi0Mlz-_Wv-eRtARQO#DrfGFSd-KmsX0GsLT(B#SHRFvq(9 z>iJA?Tjrc{t*PR{J0o&*aRNFOlF&~HnjQzUl@08?zxsgwQC@a4DbJy)k9MplM7>Tm_7*&*XlW+n#aNbiA(!VFo4btT?{s<5xQh)}jyd^Y6$2CIeDWAE-8 zU6r$RQn>WkZ0_}tC50CLzupQieGXspaT)9! z#DC)A@fPvdr0lNkYGnvGZqJO7ig?t)=7u5Ezt8W+kHDtxq@R|_N^;egeKB!uy7N!3qz-x;yaD-QZ0*EYPERy?e`ahU3&&>jV z`5v`>z$&4ME)~C*X)Em#RB7ohfT#kl0lMdAbzQLLlk{JI}iOylZRpvSYNTbCJTi$Kv+2mV`Zl$ok{&{_eM zs>XV;8_FCEi7-Er>(T(e47=+9L(1IVYjml$OJqa`t92Za%Q)-fd^ecE)?7V4x>|np zpNP@_%>o1=habKNu9iZ^Tf;8I*#=vE$N1uwqpMkc9M6Gj|JjH#oj+{>U4Zr7@n`dS z_=a04=Ryl|>kaRO0>-}MhKiwKi$2(8FocZC82<*?dw49=3J*DK*o=(J_=Ep;F;k9L z4U)3CEbiEa2lpo>W8^VUP{Q=G+uwgCHAn!O~1Yy@K*`z0s{|>%Qg$#>7is;&sCA@{160l9Av(I*in8% zClh4{l+5<%zC1NqA_5v~dx-w8R%hdN@{i%SelM`A)>?psdt)(u7I>d#0L3>Y4+GEc z|F)(RW7LWpj2Ejb>xTaH76|*x?mJZ}=S=IWKzsiY$a@HC6E7Hk1gtam^cV;2elud! z22zXMRqc{VimuLBwus{4BXB(%MXu+%5vqW&?$LODdj>34>njEHsn6~FL3?1Ovjd?4 zXzRiKF#A7HF$!m!QaYMHY1!cZsKFKHg&}2&K3;yB8$zRCtZlHE>QkVP6@7BMkp1c) zQkkfs++{S*;)39X`=$*D+X5NW-Q3~F9p7Q1h#Fv3n8FAi7D-Zu5amJ6xjzFzYM?kA z7FkJVf_?*Gh!@FAWYNgG$7%cBi9&S<%A^rI{;Nj76z^O{wns9wJ3gY&>gqvnfsMDf zaFvhxBEWvSL~?w{;E!!SAq4)Hxoz83a+_^1B+TsyCMP^U-SfI<*~CKd8-VN>VnwSq zT7M~W^k~dT#`DdX;e*byy)br7q5KWaY!Row!4aVL_BlRx76!981cNfLULlbFI~(>2lxt(sWG(nSkV31O4~}7F6CdLh21hYK+H^M+y>HI-HbOv zFMu||HIy~X&|6$DKFgV53g78AF zU!_^g(N7E@NQ5MyBb}x4aGUlFbK*oydGOEf5eN+MfvWDePFD z7QqrIR41j4?u=WZV&R^dH(v;8a$ag9Klk7>dCAb zA2dle9xiqBk|IQf7`IZteOUp+;_7!t92^eK+hs1<5I9^z!E}~E@!ga+SVaY_5pPzG zK{w$k7~v;K@^1|8Jc!CABaovJ^H-Ii*=+;}afz*A5noe8%qKw^Ebo^f7~<#MfOO4a z)TtqU*yxL{<@H6dtYK|JT>gmKhF89EsTfLLiBb7c4JOT}r7;=~%cu!0Ptq*((lDV_ zHk=}Z$w7)G8)U|0h{lj*Zi8Pz+2DC&s#9YuJ2y0sUdibXza*lNaWY++0`m*L=KO%L zp|&z9Wtz{w6&!?!e+|dwwVWyYpO}FbRq%~an|vJ?0-yg6wEMX>8YwOphBK!A;atUh zA!J1voey$*Y)I@APEpd*UXhADFndN*rn&tMHE{hu0g6{>zSLab#_qPiS~oftr|{_u zb>Rw-dv!oHmCz1Mk+SxZWdt6R@s^TF5+X!dj=FIFNA;(tZ~|EjD4Yr;D|C1s1H$fZkMDE zdjt*{qh=NcZUc^UTd(%VkajvxuIHo{!(^?o1tBQA|Sgc1I?f(`&85rC_ zX4`6Lqyi~9ZraIa47m~f6t>4?WVe0{7=ZojHhA4Um2*;C| zf%b<#ey(F)ZIoD_Wk2;fb%_+W$spmu^4DLQpxOwxC)-tkjKj4c!2|7e5&iqq&6iB)t)3I!8E!sNPA88InYttI~&Ok_N@&xf4VJ(xoFl`w$ zY~fQevc16eAI5Y9Y66`~xjfkidIsJzi<)J-7n>aZON&^f z{nr92{aP@w4)CfKzPxgE$b&)fZUDh8DBmNA4}?rG)fLt@tzr-FZ0ymkznB+ATs9wo zw9wX-uy9h;5VoE$PcvjS{sR_HTu@~KCd3g3NscoFi;(67xZzo0aC9jM3|!c$0kq4C zq0vDxOj?g-Bk5a=AG58}!swg0cV4xY2s!T?-!WuPl7>1AGM9X@zy|hwB)J>NnYC1< z>vDczn_8-(o0gDX~Y=j!hp6JAj3>?NdE5W*66^XyY3WZao!Si8+M1x zqZE<#w9%5{Qs*j4>dO?z9pZO+Au!1dZz8}RCfW^Xw|cjOW7H;dybjw#G!V@d z(8C96S>N|(=`tG{`1A$-1QgYfqnF#^$0c$iqhbyA4}hTt{thtrLOu*)ohsy8#XOUq z1mcK3j3YjYcwQ?wX?$LkxApt|a^2G;$pHmSV9-wot>F`J%&St?nVaw-oPp)w{Hb%9 z#1Jz4Jlpu)0vujW;BWX*dj@TACsP_TpeFZY8{t&lEPHD2@r3tGgJlNPyWZUw7#|p( z%(QS|=5U*HNVNS+loCEP%b+!wn!0gf`;XXl1|ZF#_3mOIFg7nrop-(^9P)d}mW3sd z06Q#pR@Zf&{>)1jH>QX$+Yb>3X(nAz_lN1^>8Zd#1JTav2$PZgREYleVnVYAe{@F2vtYd&*2^liSTD7vV# zohdO|YVm_yE+Xm$Ct4AP5`6|Jk!UE@C-3eLjh0rW|J`IiI*Bvq_2pglTSa;k1mFuxd7381Y~gphAtyFK@*V~SB-+Z z{9X`)?#cWM_hyppfazQyUoAyw{~qxjmK>KqlKE_$VAd zXNBL>bSbkP=5Cew2F(d~E8FF-Mema!j0uLf{d+|#K)u{&K_Txq=_k(c!gYX^dm}!9 zoOGx$4w0(p!EVg4gZbNObr=VzGzczmnNqbQNos1u{$ZE%D{#yo!+NH=&&k~Nxv5CIU692Gd?iN_j<;i>p<6@03>x>M_}li z7*Cx5XsarP)(hZEOtFseZH9q)`mjWqaCZ7_7YbPG9+$+aSN9D`_ozn8mCTcmQ9+}9 zinHg3MF}6*8Jv{QDaB8-UWEoc_P|0eZ7~|kE`jt(%YJ4=6n5mM8|vsU=K&)@9v6gJ zyv=|YhqMvuhX=}ptFN+DG_K;%Z%nt2AsIbUzICH;j=$^) zz1$wnvZS?t>)_@v@qovmif~zI0`-gHTA>o;iWkmrV?lGYEIwe^0|R&1i>@FFAg&RK zBJb4AIA#dJzlT3sL+sB~E>P~92r~u~ z(#5L%b6D6pVYf>cS7j1Xo&3xD{x9PNFw|0CaERl36jHkbg{EvF9;GS_$Bj`1n^n6bEn&MOttRzCPDZX7$NOLLY5{jRSi&HrQ zXAvL)F|_8(zQZz@m>YENgqinZou^XmhpK^PKT&hZxY&3veLjY6gt&CL%29|zBQqLJf+Umb9)^}x6QV%M*gq&|kr8ew9G&{Hr+5v3jb#cY_=p`0jrX*kF z7vB2Dyx`}c>*-Zcmco$BH-2oi4Lnsie>M0poL^wof4U4at=LE99f~fI-v-*XJHQbD z=UBC; z=#;e(obfe-EJQhCbiEk(N994gNj%c*VNFxw!XJ~BLaNW0$OB{FzRPUsQ_O0gk?s~@ z(oy$nAIBo$pIiFl*m}UZ-a;!%b>H}SpytGMs4@NAi{uAkq?j{qVkN-Wc3AHS_Myz;JU$)$}BxNjS zGLY#m6f?BLCwBFlv?{(exk6HYat)BaT29E2$JP#2ycm(?Dz82q*&JDUoCH~qMP5ph7M*o+W3r>Bb zpEdkEXA~suL#($SDhl~RlPmKM*9`}~>&$T1J z?90(Ly?sgjn z0BdAV6ZZhf7|n~pRG^(8jAy_5+JibZxyi*a0h#de1n!Nd9mwxZJ%adJI)Nrp)C+x| ze%k4>o)J^8*n52naknRDy9jL>uJ1t5WaNk1&&hqFq;5`VI|DhPKNVS40OAAEbl4y*u^Qda-0V;@M&buX8 z{C-<)IZqOW&vscDR`CaotKmqJ#BN@uE{BaIW_H%1Px?mq%AMzfFnghB9ano8=B zKmO+z*gTB~k#2buO!VJB)1flR(>}?|6$EV;12gw{w;-06Kve6|qIzU2)Jo+|3=(Jg zp!m}nXrYbZMut{e3+tL#;4D5$CWuC9nUUXh!R~yt)s5#$jd~3f$)^AyqJl{X2}Yc! z?^A}F(u+-gX|Tx>_{PnZCztg4+V5z&t!4T~dZNnTsP5l!%SlKZw5Qr(OU_Zs_k4g! zw@5CjOHZk6)i1Es0DwPdpy*|oY@{+b0AIN(g@_;cn@&K7=d#4XqXKV%qcx6N=ubf1 znW@mVT+(jyxZmKf+3^fQMlOY|j}|$xzHCF3hrC z`}9vK5K?n<{?3D*eF|d%*kz&2D~0ojKd1!9Hy{%<{Yn#-;0B3)oCD*>qlV2Wqt}?1 zS}2rmx!z;6ln`Zf5=u;C1rqPL(p+hY6LB#qGX>y(S)Jbjw7oyCyr>hb#2T~|mD=As z88QpQR!uvUmj1*HOzpAwcal!&#u^X{)pSqbCQ0fT+Qog&{}(VTB?V{-yd*KIxN60| z4J>AFNz+$`qlhKG0ZYO?t0&+we$`GLa0b-#yGqB4Dg*<)+{H2ZkBn4Byw-o`NSJC4 z5sQFyA&Z=ak0Oa3L3?MefgWeS(P!;Vz4;0R2CGr1K_)}rtJ*z${B=gQ`0G(r0u}x` zYDdBMi)?bpBZwr3gWLmT=oc#MSG{g2F|$VC+KH8TDszf)3kEjVa1nhsdww)R5uA8N zke^UWvnffodmpJ;YULp3Z*n7?cS<|M5rZFj=@g_~-tbjvH`pA5^utVGV^G`@^>?%6 zG>FK2)SFI=Y%A`*Y(Nt09mvNpLiCxiSd~cz-a74$1Hc^tv`Vq@nh^^$c|RQk5DC=g zBjvS5pm(ASMhbu3=8kc6=tTGn7!-ya3@*y}hP5&W401RU*3HW`6o#rmPMKooY%>5l zJ>#vFt58{~lc6#6pw>*Gd`^s!v1ITs35uu3&LA0=I&_CSdLjhx=UaPUx|75}Fg{3Rje= z*3GjYBGFUu`qs$uPdR*|$C}&&Mt;p6da6m#3g*}01nTI7o3bx0^E?8NL^r!UOwH)g z3yn+rJlLMH@!J`ipB8g9^c|P7Gj=I`J;U#{=bs%@vZyzvCcDHVJX_uxh21{ydY~HQ zxNnorRvW!%ITC$1?Vgb((?Erq|1#jEe3GN}fGD4n3dVxY^`0Pv5leNgVcDY;NgnDp zDtJ()iX=D*aUZ+W|Ji8Ed$_42a8G;Va?-N&Vhjfdj9_baBxA9A%*k;_&nMDyvN2ox zVT!}%A*J*s#2t>5L9SCzj1D!cn^?!re=oWJDGOM*#Z0B388OYG;*(!7cjBF5Q3~@P zZiOKz=WfzjJs4~4zLT32(@1R|1x#3p}mAw(F*H86Pp6h8SihS2Y0cS3+ zi5I?VPGprf5>xJ(9_>e}KdhUw4ws+3@`+dct!-~eNzcGSkF|R`&K;k%c1pyTBiMS? zC`t;a2M#v433#5j(Gsg1A|d4o+|!^o!jGvyav&vzQP*PcyhxlA0*(NBFeIw5gn6~` z73vyd-|d6S_Mhh{S?cEs=h=U5@kOa4gQi4{;vo8Yk;XwEGq)*bfQc@{f3-V7cUbhl zFGR3w&7bGceD->aoTzRGBn$?CLT^~1QXn(4KX$WKzb*HQ!B+rW5U~2R2UI+;d&r@E z0^|MM15^+FkS&y~$+>EyB|rEV3VgC5;b{)!9qVvQT~3!k@Sp{+@B%A`_HBfW*Q%;U zO{a4ci>(8AStcb`i**8L%AamtSU?|wsR>JwFO3UtdwmTZX@y>o(LeXU{ndk`1afd> z(NZN$q-xl1{Yd%P+v)GAyzAyw!_4z^2T*YNRsd30=L1yWs9R5|nx>-wDTR=iY#*@t zuNmPh60%9l51|#Wxa0xx#`0v2_v?mxGW{s}=(H{}+ErL}#^BT8k5SEMWUd50OT;+s zEcAl}Ue}t9W!pS@SZw5jTgg<^SCKp-eBUw+M(UREsevgyh&=c8`X>lLAUb#_EeH4; z_E349`aecFQgC6jY1LJAmUW2+K-yj+YO@0C-#jeVN@stSk23oba7I3LPzI!dbVfs36(d!F3i%r(wQnmOhUnQeRE zdty@;Fi z2!3IE29_hH^Ab8VATWnJjSwN&WGER#QabW3Djl{41-J_Zt-4;fndAdqP~FD|3T|ID9)gKm6Y;z#iB?xLT_2B)D!;FTY)pV5?#%9ggvBtBVhC`> zrT%z20O#z`p~5d5xoc$qv=!W*O1HX!P>~kK>x%k|oiV<9-$m zQ2XD^;r<)fDPIlK0?KM#Av9k=YBJjw7^U-uTnZS`e^QCChN6-GCJ6_yBO%1qj=d8) zEP4p9x13GSL9KT)9rXs<)iDSr+qE1Kwf+12x(#UzKpC4C8O-AG6Y_$_-o)%W<_z!g z$PFW#YT-S@z`Fi6Mim}n{teR&Z9sx4uPoFc+ln}vhgOFW)Gi$O8g733_a+5Q-NDMl zXY1fPfcbiFo@A)jfrXmN9vk_%3kzlpb#GqoO|g~s za9`$yQr7Hv#pUMFO%;`?-h%mttHTDDcxx>eep=7TR~rP5wg9}7Dofv9g7xa`>mS&o zNvb7+ij;~XLpDg=t~q@( zjS$-&(q@9NgpXj`JYF2__BU(#zoz;B5XHguD#f{x4pTfzbf_Gf(#&m@K`a#!xF%X} zgZ1~GhmGf z!~CjbBq=7GpEg>xF;~B3|G!fF9k!l+qMjyz>73`}2Q=5ec(80t{ym|)~y!UBM7WkE+* znY_dPAlIXu(aRhF)C{yWQ-d&8=Wp$6G~Ii_yq~?|s+M6IZ>e5s#mOu-2yjopok5y~ zSUJ`)vt_}Z_8RyOTAhSfkRoc_5~1i^v<$3kd z`tDqH!2z(d=H!|2tW@9fQwO|UOT3Iy6%_8?6*E;J{7Ke|^6zDvjz!bI?F{Qf+)#5o z^1n$DXo584m5*{eFzs$ewo`{!a1Pjg;1-5u3PBoC(sI7^&G8aUZQALJA_YIB5zz~~ ze+aHoF^mQ?=8di2I}i>fDA+>zT|{Sp@72m?SY_siQ+ylR!>hg57qlx6)ET zi~DYy!lA!hYuSSYz5z;_Vu17VjlCb-4-hx|X@f@rjGbgOghFstV9BYN-*bv4)LPnp z$Q*i=%&Li>D%1!b%o?^1?i3sYt>P41#8M*K72S=Y=Kt`o4J3R<0+ad(z=Pkv{;+O$ z3KB?Q&q;JnA&cY)C+4FZ?>4D>3?bg5xF9GvucpNkC8asccO8VWN_IO0N`6da)%@kv zZLTQ>p~&1}7-X!0A25M=l`l8Q9bQ8gO++ggr?|e8 zcmQ^FrhOkN?;+7|H_RZ5cd5MATI$clD32hqH0KR|tl(XxsvklTD~*R)w^)uE{HtD;?r5-gJ!+vqZQR&RkowJ_d?7Cwv%@kc1KrZJLF zOO@nq*0?6w?tf`AB<}t%L7rVjotc^tVgk<)3d@;F&C@HXV!mAjHXJKm?<`2})>uyD z2jj3r^G!meiMg^#mN`*k{CMT!j!dl26(Sg!1ErFdxdQ3lC56C#f&u2Tf?tYQC?o~R z(!pEbTi0eGw3?uHbB#d+@(5-TLRU-#rE=q35|oK@Z9YE7ZH0>|T%-c9S2{c5Lqfof zRuF7^Kod^n51(RtosUAw&|L5EBe0=x{KtH$21lYX?pvA(F|v%o!J=9a?`Q#zuJ%uj zTW~yS4dB?iQ6m#4OKiNr&ki(#7Pzq7PZI0AH`z-`M9Ym1&BrPt=$UluF4JohJlrB7 zmjT%<(u|5$9VrUiUKCm6^C*}>34wwL&|f#g?D#zcv422BfDUHJ5&{xP+u?*`_F0f1 zB7hKbwM7>zJee6y%=*Qk#7>01MiR`tBUmPF(<{J;0T%*ugVw|aU}Qg(_O$pvdmy0w z{nfQbjl>|x$os}q?G`i!2{H0Poh(=|dwh&8y#&Ls{qfonL=1jH_at#dHvd!mfj=xk z0D!=mUnrNRx}ZzfCJ>DIgs=h_@N}UnuPMkWNtuFzhk+SsnS^dSXsC+0dy(yZtqCrH zHpgkC>(sHlM8Nq!G@W-m)&2X&%bv+dR!HaA*_jF1$38}sS!T&z*;Mv+>=}+IMJQyI zB*}=}*%FeKJ+r>or^oO2@ArG(pZoZX_xpOkuGi~%-PZj|_KGrGYVLyXxJ{ljSqmse z;+t;epeDQ%&>`V({zw>SK(4#yK~AZ5AmX_W-`Wkg3+^87fzcc|fg8Q%4Wj5$3EyFx z$gF*oEcXuY3h_@G|2Esu9qqeq#A!^Du9ME2lZd%2^@-+VD8mLN&l-3b`Nz_XpC=M9 zB_Yy*cm2)X2i&R!=ImC}q^5+NQ{2S|VCKt`;xI#>IkT+O765NxsEH2AS*{|j_iDOp z?LQ1slvSjh22iXDT$n_R>v|ZS_*-(_`tqIC!a{+0xM!&jSZ#ec_58Em(YIpzr;V zA8@-{fByI&%3TFtml9S>m?sz+uHAauRy*U07%dgAo%J#e1&-p>9ZiCZwdzRAjRQ51 z3m}e0C-JWI%d9(e^w9)(zxRDH<@4$nym3zG#qTec4z7TU2Ab5rvKTHI`HwLYuo2|7 zg%^T)lSzvX)?{kg5P0Z~bTtGT4DNnlB$888{Jnx%x2r5ZhA zmTJo}*^7kNUd+SYp>vl%tlgsJ@OZJ+5tgVsZ}inrpR>L7&O0;s z{1k|&O8G3Cir?HOtfsT|T)*<%pg2N!=HAyZR`X0z)>_-obYsHc z(8+kAM8g5?=P{O3>r3$d->mpO0fdtUBzc0_M69|2_>@~IXR@>P*c?%XP@i_e(mCmiG*iv3m=VS>jer z6O8x?wH|u(wZ2))kha6kvt0W6+33u{-<-!QSyqy!-WTQPe-h?o|><+_i=<3(xm?VL(*Gnn#i(so%6YhnGG0s zeNyajpT+yG&LE;f+zDwOwd<4-XBxYajcUQMFOl_rjnQLbW6eISW-)`+u(|XEsF~l# zA3o3vQ0fxr`xPiWt*(FD1x1%7p>&vSNvG9Ea_%W=XQz7qCHB{az!OVpXTu)ORR5kU zL#8*Jj(;nposqzRv34g&9@(6(zvIvUCi;CHTI*6@PTdr-0v^pS1SuK;Jm8db@`1V| zzC+!>-Y=JKfUtZ^(AZx;Egz4e$^U+6Ig<9gHud@1oev5}LFP3^1H2zu-@5ZEjC*{! zJSRR!Y+KHxZ86Ank=CM<_fM7bLU?RA=Jr0}MFPB$rr^1~sRy3|#I_xTwQdJ1)exAW zS_+`4$}s;evMfiralHPaQW*B4n=-P9PbN|Elr%=3ew6F-D(-3bS7?;dg&cuLG4tpu z$Kzz==RBJ{*!tnJK4evYzBM>x=?C7SoZ^cvK9~a^83FIpQwYmTmZefTJH?ywJv{40 z%_gm%uPGX#8110LUBCLk=iGPxzJSwzSZAIL)F^1622RXJLj|>^sXR}T>x#R6G7+axJHE z`O(RLU*Jevuk$(4N+$j%dD`E`L$hbvVMa-2{ArWZG(ztsuIUTyfu&Y!d$Z&yOGuU^ zG-mI*y)Aqa_M^A-IaNADno;Q#f!R{7E~8%u3m%+7Z||>xCH)rT0w~9Cl1<_G<@cmO zFDvJWr@T^q5b7Kv_?K%0>9(h~8eHdbonoR)xMjNAhR(!o5r%4S%$|YsWkC{HF8k(?@x${P!v&+>eY!^SxQDOA; zgSmG2)L_AWr|KoE{1330Wu~Re;AW~UaQU&n#SJg1#j~9I%KsRu7RAB%!Yz;`#ryA4 zR0|T4P>Ik`PR`n9AeIa*8ubeVbh>VV40t@3R+^Ri$I4uyZx8tgW^BtHjV}u(VUDha zq+{@Pw(lP1M=I>LXF24XEqjnnTdl}|lT2WQEJQP{onndZlDLJw<}#%J$NwIH5H*RS zn3wx_k_quqnbAwnIipmJ@~PRnK3@}cH9(%8pyPkf((-)HdTmbi^*KGwvtOCOQDnYW z&TA;)S>U~%{tS*oI|fYf*)1rY+l(3@d|{Lxqj>q8B4SvT{BRGvIb%e|1_RjWBAJ;u z@d{LrW|jfv$J6&CuBIMF(lS3LRw_?n%9{16bCi7nZ<3KF)lYZxKwNm}?^#Iwt|0CM z-vxX|wgUFe4mq1za?sQb?r`NWEMrp8#o|Y0jfLPGF4DhSXqeD<0l`Ep`DM;h;6G#6 zMIUU~)fRN$daE!ph3Qi6Ks>$HExOp$pCv!K3lx4-YY<(<^^(cP5}u1iAQ4nTs%~Ju zPC+{3x}Qvm*F-)OjqtGVklhrmXg5fEr%7|9V5pxSLrrI>Ct<^fHG38wSl)@npUBf3 z8>K%hE5w-bpFANCuQ)4vjp&&wc)YKZ^)_FkD6L8{UED2CF+DcBy#UK3b&-~!(nAFib&&sx+?iZ=Y%g~ z0q3H_BqU~-q{b(&WsLi2&99ZFP*(IPiQUMq3v2hcx!MRF`GNT94^SoSyE{rb3BUC5 zG&^yff1+F=;_hXGes$OpZQ%D~&R=ce$H{S>U2+oR*ZoNCVpLV)+2LoD5`GN4=Lm^P z_e^X9SUAP6`x01+;ELuQr)t?AN8VuIdy&~#fAd**__Oe8qulPAZG6`+>L;Ai5>qZA zaFlxsPCc1B_F+TW2SkQLChe!NowhWLseX*r< z(1b-nL=1~Ww09EEB1fQkOxdlvm&{Yid|eT+Z)C?xji9@H{X~*wCjg`pWO@IPtP9TF zX3q}uLb^@6aCmY;P9gWC6!;C+MdA9c`HmT!A0DeS1mUWCu&;KsFNBnb@^AaXH=??a{TP&bk#p8Iew7JLrp@>5XGy#(c2GWb zK19r{RUZFag zwYUWn(60(R(+0-CQ~#%-#j&4j^ZShT6tQCt90lgUy&|C3lFxgM0vAL6lu(z#HVvRC z_`*lKP$}OsiKc$UB5C2VV4vQg@iI=t=9}WM6|P<(4bRV81~pGdLG=y4=_EX67hmmc z^eb;X7jHLFWwCo=sDG}5yug~wW*3g#bv}|}LowT5|ID3ls*DlSZQl|$rqd$6{-4ii z&wrDZ35_#{%JgP*lmt|cwD~+T#qB{c4ooa^X9jSiP2GoUuiZt3?~aDAC5}&KX_4_* zcR4UEqXI-`Cx3! zwmgP^e_g5W>5caSN^FsY%LKg2hIflmH%7DEaP9E31j|r9knKKndSiN=GF)`7wLZY| z*6hvKUCyLj6c8CeUD3bU+9M-1Z` zX^O^ewS@!xkA{1~O&#{H4anETqkdYx$ErO-6eJIXI*63BvOa5(>){Bt3ZQ!J7!Mq= zV^&9?@D1Izh9ZL)N_0Q=7tJ%WvU>GB`M2>PRyQDLYK#rEl55W>&b8qk9?qZDan0*{ zE%XYdRM@^Lz^Bk!Imx>G*YYH4$xSzaGUbr!D+nCQHOz|i0&UQ%BUke4fdJ}+B>=$| z_7mh*s(KzS)A3o5eSA>$_w6UPG*H3K0$7ve|!RGjn5AX18hPCRQH<0h1k<8G_>%Y zVcxqm{;|JFIoG9?LNnu3BDjYW;~QCkD4WZ z(!1~IbD3V;)O^ZabT`|#D{a@Z%HnLMiN}3zrKmN7LL-dd+NGvaxlh9L)y0Nya-}M4XTJXfP^9>yXK9Dl(VZ7Mx%wX=bi$io=@^JIY|R(Q z&hUU2p<#DQZIq` zvXUI)<3U)kJomlQilg)Oks6di*GrR}dAVF|I4jHbDH@*u>+l_A}%a7*pMnHu9{wjxXaOCr;Sn${|e z(Q~;Xz(n)BDr8n~uGE;K&be6nhAo2kTn+(Iq?eQ1x7N*Gxv*C+ewuafJJCgp#VV&lMxCCHkkNT*5NShao!VQcJAP&)2*?7D-=2AO8M-Us z7aPbbG{+)NpXre1L|S#}Z~lxo)EvN~t!$>H38z0nzxf&QjfLW0A)B4fg(G2foP8d&m!a=LW`-|RnaHxP`8mgoEa zJAUO{s<*w&h9b`_BElqoiyyCewwjTr4dzw>E-N)@{2&cdx>;U&+4rk{p1> zz$?B=qKAjBii-VLo?E5v-0N?A5NbAY$cw z5@1lGvtRfFk&@ShT*ZZ*Tj&E4eSrqnh2NGyXhH&_ybehg z{I}jz$X_k%Q-{xTnubNbt10VhYW3r9S5`^wEu;77)E21s3H#(Gm2D+Im)zlEHm)Kn z^vsHR6a|pGz*DpWHU33DEORzWi_8}G!IXVIk3*nN8uv(X-R&=KmW zi9*nl*J$;^g_=b%{J)NM8KXn_x&%$D-o=A$Yln)~qE=x46d}XsaR{khUB&~C?`eU{ z2s%*NuWafP9Wm%zQn)UWWklqW-}eu^FlX^IQ*c@n5Hr=865o&xXX)-m>$baGi=5F@ z53~srqJ-cVhE|Zt#+OCCr3~jJ@8iL^;WWusT0qFuA1WER704{(Q4(EQv#s5g{)s)H zySfluJeylfG1g>7Tfn~%ky}IX!Dm7t@4UP1Z}~p17eN4Oo2k|s6wcDu{tDUZS9e(% zZ-AVkzV5YsTP=g3XU1Q&hv%s-*3ctLUn=Y%rp2f7H7yIKow7=`<0erRyLeui;Pgqr zXgGc6nGZmc-%{XBsgQ3hIz{>)^xo=02ZuWl#r5^&Ed6}ge6?aFtv3@#SZU0Pbccri zhH1T<&D4_q!sYy_ETcxB`*Li1?;&<~`|r~#^HtT)>-Qd!tG%^NCTo|Trxp1~p6`vD zuq8iB!@#>wMj+3=Qvb3e4DtQjcR**b6TyD^Tp|sMxlzanQ~{PN7{F3|05@&hD}qUMVgKRe8~MOBx{(Ib8UB#9xn_Kwg3@+8_uITZGX zoM{ayYJAt$TIP)6(28fR&nxcmFTPb+S7xBV4BPVS= z(XUix$)6zO_fLIgG8i3ToxmNAlcy&UpB4FdVVSZssY7P?%N4lQ+K(yDjlh@q(6r3-TPyf&=k07;`bC0`1-G;;{wHsi?Q2ip6`N=hmHjgL#DoQbHQz6s6+HZZFTjNw-*b8`ChmoX4@u3SIeq_I8}s?d5kS8b zwp$s3sO$!Er6OLRnRVM-CeU zT$aUdIQ*VodTFo6x|&{j^LP5;ACcYRPV}`3p;1ArO=$S>iasI5kPg0>=RbO_J^W=Y zuN9D+|RC%h5fqeY=~B=YF%liXE#jw~hRJ zeCKbyVc*I$%c|wbjPXapo#%_j(@v3R{R;=n>>nI-62EL%6q5>a4)NXf{tf1!KI<+y zvj_y+J21!e&LC3Qh1wEd=cvs2r7Q}fk^Pv@2BD?laGH7xj_uJkbK1+D0K0aOm9}~7 z2UYyoffFz{TtH=_@q7I7_i)g&^!byXEb&i0`+?6`kG`L++wA@Cs%6?5>mSVNHZ^f@ zhtxUt7lE@t2Hu8BGfwNscct4vUam2{eGC3*>QM^WACsRHv zJAsjPN6;DeQoBO(%;~eJG!o3mdbddY5Cv_ z2Z=w+6|=*boZJY*dc7BqrjFme-{A2x&7YF%9O~mp@TGEw!SxQM-LVOFH zSOgP;nobaf_nGARsYiV|lK1~vFGxw2Epj$srE>(PCQ=7QC9UT?7bq+lC_e+$XbF( z+l{Z>g$`Muu_JIp=&c+fSn-VYO}s%2clQvN98-h?_6^ih1NL;2{!1?)x?hsa-yF}O zFp1fWTd@|z3dOHXo#NFB-7B`>Bl6Rq#gtovalJO4k>_Stg5R3RpqK)Gvnoxn?qSoK z02E(&V0wLWxMX!w{qvjGY_#LM;bbVp%!)rj-rw0yssimn-fhN5y~z@_2Z<{prj<^O zv)4RMhjVmE1-63A>CHDZO^+9BqwHjOz{9vs#2Gw=LxH{C`vbvqUa@P1^vQ%5l?Zm_ zQy!W()iKX@TYGKgaJ7IHH4F)Hd(VHp=unN}8}`!t@QoH>klp-Ed5!;H*EuLpsWY&h zxK@~~?_hiB{0`P59p@?Mi2%%P1sdqv=eU?02D;lkb=%>6y7->_9jhz^bVp7X zEN4!;G^0d0*ChNv^0%D4O51U;L~89cT}y2BcZ*_MVR`T{nA|Te~RSy8A)xT>rv1&h~_wb8+#_c zMQ9O@7N|q>awEng2d$N^@?D(omh@+YLmmVmC2$L^2Z9$gUA3q&jfUXipv_ZG4!rpj zK>bT9<_@W~gYEWW+rs(P^N$i6sOZ~7Z3*q=lWj1>x?HB6%l%L2e{J*%n|68md%?1jvrs#8+%=S!`0gCIdHh0_4xu z#sa~J&-Se3%ndEDZc!1+? zjMGFiz~E{pd$KB*^!3E{d!}s4!{!3b2nF7!Dm6;(0fd!KT@3`HW_2#3x1qm4x3Uw^ zpK5!h%`^DufBw}HXymejBe2d zSuaUX!gX@Ign-KoE=)GhckS=|B`M#PaR@9>fM0`R`X7LauE=%tou|c^F$q9<`W zOY5IFAZ!z6Xu5D&0XGHp!u@bpFM8z&D$$2^<~iE=mzP3Drt<@R$Ck%pcP(dGq~t&M z9QLuh*q=VhN4>e{m44vy`i}JFOho9koEfCVLPxUw%wPhA6wp^O~< zDm^l7#@o5&i)4s`_rRFTdhWu?!cb{_m)vZd{mu>F{T`aP>~&U)x^gqlHySYgn{{_@ z96XmR>OXV3;<9-sTmK)8thhKjSw-PaJS{h$)+y?3UET*=SP7`9R>31i6~G)O4)?Z6 zf&Cf!_=KKYyYo4G;gbgH7t(fN4fdXS4}IFTVh_$g_+m>V;)H9ceniQle1> zx>A$NNo%v*>0J-YU4>(B26cJ^{w~~gJMEZ~W@9=rlp$Xu)xmP?5-*=7MSm&p&dZa@ zG8FnwDjlCPo2~W;|3DN4n&nu%J*3t3B+bsOZ&B)M-)h#i%B4kFhdW2XdiVTSez&Zy z2L14x={pJLJ=lkD>SP~I+H2d}`xcVki$+#f;xWe;9FNV*vG@XLkumMyH53_+)uVAg z*;0{dA9w7azOH3@P1Gg0<%zy;k1ZX(`vY=3IoFRjPYOs1xS`~$QQU_&_?1N}B)2`G zdu_|PnigTd0y?%?BI<`;iaC7#5jjCX#52q=-ngXC?*Z@f*dufM`L)47i0duXrXJYY z)7{~nU+D63w5CH$Y_PC&iD@~tS-wG7u=0@Rk#sr$XzY%BtdTiOb^}m{gOTAZD!rVk zR(udHWJv*Rx@H};r>cbU);^Nze78yt%`8M6df9ftvUKZfyJ?t2vC6yeyQn_xE+SI& z-XuWzS;wJvZfnVXcLe{8BuO|aneO+2s-B;bh3Mw3j^AnBFs{my74H5!xKC$4Ow_;f zB)WEaJrYEPSm1N8+;ix~HLB}ih z94rU!@b%QZPY$FZ%2MSQTEI)O+XKnbSt}ep0vBg6L%ZB{a?;qT*)pNq)M09P_r~)t zdA&RIp%jvKWB3zInWEu&w=u>pcbzjO)%97_xkBNP6mS#uIeaKb7)A-PGpvMw7lLY%-EXPY_PmL74JLJ>62B}n4|J?F# zPELzwp24q{ibLkReE$drj1S>Fy5Z<^^3oN?oEo>S@(&sDQZ2`QFk)snBSi|+br{4u zCSIQ|IDk}lU9a_hl(FI%+e?k!OZIrFe^>4>*@Zw-BiI5S1@VPZR&SZG%w1kL@dicPW-fCX0xLzGaf~P0n_EH zG&Bo&5*SmtoDoA2BzjHw?I#&z^$E=65@I%Ij}ZPI4e6)S)NpNbQi6fCqf{EPIx5O0 zxNmJOxQb-ew=)THf*M8T^^Qrl!=1dz%{{(W<_x06@gFY2yH0(8Em4FA$)+;_S-a@% zZ!X*v{l&9*Kxmr8G{USI3#6I_0A5|O*V!B0+FX}3OXK*)r{hzEzgUl)+Z%CMH#^f^ zPub-R(wA3d(?$%jwt-B4<{&d>OTwcw!sAT|SVW=_3337n!RAALq)HgA|9H`0qb+q6 zy56qOFUQPv>$gQchaz{##iW}eC2z)yiIn%>p%K5#pd_MnVMeIaIZ5?)k}VG99Wekl zQTx_Jv(C#4uzHgmFSQ;Etm{nJ;3hF2oG%cd5{?GWYH_rI_oJ<)qC&(wJHoq1yQ>!x zfEDi*x=wEZe^Rh40E)c|bM1HSd$DDji&SVef-)(-AfPv#QrBl@`&3X81B?W*TWrj~ zH+5148>=$cE}ZYyzMXi!aRucGKJjXK$8oaaHyaU4Hs@%viWLrj(nzYL#bJ%mueLhg z>F&5KTC4npo%1w{j!a(KYy~zGS^WzC0y6og%pBs?k#~Z2iN$b+F}LoFTed6A1+lTh z2tuCXgUg1&-B1Th8qZch;$Ng325m4CKE7zYlp4X;Po39vHO~VRLdkIV5jj0qNO7<6 zFkG8#@^~Yowf-^X`L|G0HA0ou0B(Y7%U}4<{a}^I^6=LZPxVkPPdOHfNRDW}mmEn- zU|FqUOPa;~1K4@`KncpCG|1Lm+*N;) zY+PiMZCQAI`gdp?XHAmyUp>a?!pklxM*cIa8{nY2i&Bh%g_M@Xs}*Whxj!iq2A7-D zN@1_`k&U5O&#>^xYJ^1d1RPhdH?l{ZVeHYC-TJDm+?ar}$>gLydVKZg7Qh~d2X5;M zU?TUVt_fP-8MlyQXiw%KxyiiQayM zNK~xX0{yrM;!13BnSL$)+jqaPX!*lU$U-C_B*GvMPs+wBrgD0qc=i{(aBtG+I2+rE zeq6V;AI9PlVG@iE*KRCczURV;X})B78mLf$=Zvr(;xzDpTa( zHB&K^>Uq3Xic-z>$2BcO1L#e29y|Tqx=4T%_>aSyIquyzoqwLC`F`5|GlzfFAu_FD z^s7ecRS20)9J_*475Ly!na1^nc=FvXt_=~c1;$l5LyUW1mKdd{dEwyzOUB8 zxpWG_rr*1}i~PzAT!4_hSp$=uD(TwezGMA zgG_v3GDri+H=N5_$w_c8E{X|jN9$q@eNza~Rnnf|$&%b%1afw5vpbUTb*=2XCzf9b zYFJBisUK{3Njg_~2yv7VX&!fD-lemDfSGBl(fMm^y8~m8ySYNA=mVRPVfW?(IFsEM zv7l>jR{yMZu=D3~qfxOMHKj8XXQr^rFp+h3mXWl!>{GpRi2keq1Fmmx#_!ZrSVRTG zh-$S4ZX!i!1-4KRCb2K|B(!jMjbGfn;tO#Oo;mxhN9<@}$SY^CirdElQX3@hmD+E* zHQ(+w6bi>Xb1|dsk2HRYJb#98U(a>5L%CxPs>Yw$ao|i=;68kMsswl&+e4gA3fJc* zaXgXEge{{6{U?h7_;_uI2@IQM8aicJ#HbIOYu>NB_h?g)hFz8V3ih=OGqqp$y6Q>( zCLzvGtIb+!fjG~Fc-3LJyeG_uLq1q81PSzaE|gn>qGKt4?w}Bfp{#-PGufYN?!tu9)BF5i%Ax>y?aoinJPGwlP-vLx5WXZZ-!wS z5M)k-!K!dhoEvs7cvQNALbmx(WNz9}nZYGF;*akz;UBBrw6hwnjaQ(r#@#6_KY&)m zQICSd8|JL*ViC5>~aVa@g@Up$&cGL!xFFs@BLf>*jeqF4*xmV*@LEP2 z(ZLR?v5Cp25X>^Gh;^u43c%Vf&)idS6KIXOwGgkp0xPd`!)pyq0Gfr9sN_$BPpknd z00^rGyFz>8UzjxH@3(>qHW9Ze#hfRo$Lb=`TKxEFi;Gs|Cqk+@7Ttb*Xflp*jU%#7 z5|EQn#b66@XCcK@Gs5A1e^ECe5v!@~ZQsgtBn+w4$b*8DeU35O?~pm!MH1dY56VH7 zv>Lu`q}w3B(RoQuEb!wt;NNxgxoz^ZI5OG$Y#!C)CXoR)Be&FEMB&uJu;Ep=ZFZJN zd{X0Cw|S2kTWg&rVIgifa<#wQjLa{C=TR$4AmQgw)qupd{NZ@QmARqhreJa)qYXGXg9G^^p_zm zkc8+WN?o$GXVUNMFZ_WbWf2D8{qYhoo94To4$h6qs!4R|znPM9`O|0xI9yis39BD{ zhTCfw(@oSYg!34PTU8BCpE=7b4|^jrB(I9Tl*xI8)zcrumzO^brp-Qv|M{JO!10DXKs^%DKA(oG>qD)2X|PsR~hFb;QLm z?!&1j5_F33d)4A^#nESLe5bz;z8J`<5n~Aw9SQt%1^~vpki#+4qvv_Ghp)VrwX6-Z zP_8%+e}$=e?-BW+p=e9(P2y__r*j$17;%T*+JZEMaR^I(lwl!G5oZhMb;1Ms^G-B4 zX+9P4Yw{X#M4cpA_4)DvH?8QO>nA`qnOdp))VJ4iUh~$>oI5q{cjKL%ukHEi3T$M< znQ(IfGlLcq|2JlEMA>eCF5Qv8RP+G|>DMfr4gsrm!{n}wG4VYc z_JC#48?a|s{djmR%F>t>@{z5yHd=WhzLKU`l8KYQT6_VP{!{9o%u2fQtE4qx886&{ zGs0_m*I*goJ2{s!;;*`pO*H!Y0-$2+&-P5z$9~&nMaFXYqQjbRz`5Dkh?IKQiGF*4 zl}C$tzcBe!3Lo5n4r*zkFGk;3c$53B^@f26#N4?g{vt1p6`T#PIc@%KErd1c;SkA^ zQE+4P9Ul)tDI_FGk6V-@&v+wQy7EOQI=JSHp)}ebI>qoh(b2=J8<#_fkC2wwDSf3L zO+}+KY}cc`p$vt^ABOKBwOPRN+X8q*&Ziw&5b!D^XEpYrAe{pJ<7!AkjV;o51-e_9 zI8h-8(Yk!DNi{w=Vf*0=`~pQi=+J znD+_vyrl|5SyL2qJnrykSzJ)iLN3%<=E2h6Hy5PY;x&Ng5F> zjIvPxiAzs=2n(CNT?68(tWMB`v9Ju3L|M9e)~02OwGvk=l0-{x2wpa8;&s3=kt>p? zdnf@5D!h{KnN3t&z@ zV7+f34!7+F+&=A&otdF8oX7tx<&hZl@X|_bhPE|{B|8I$@^pS7d*GT9>s=;eC;sLM zkRcfv=ETK6)8L96olAcf zHG*P4yK#Y^e<3wN&aeyeroALf1|HGRYGHK*BmJg#=b9?|s893()M{E}(0rylk5KZj zJQa^`na7zyqxM5f#gB4e;9v3H(^`AL-gssc2N1vy_VMDd4@%g+`-cQl`T#cXN zn<2AeW7XxlhU*E%R27qp+-Kih>^--nix z9~VWYi8zYS?3vSv^p~)EgUsv!+i?aARPZf2e_kD)u45FyFZpv&GO(D?jqmjWWufy- zp|*b_w!a&d@T?|QxC^FlE_NhbxTkg7wc!`z(3|_W>fmu!34cMV5dRD|NTYj`6qF+T5hQ&KNLm_i6v8+X|GKg_iy>xo`Jx>ZH zF^10r5mL^Gh!Gp4&MHQqV+`!oJo$ulYAe=r!{TxAOjneY{VyW(hfPRkRVKk2`NCbU zcld(_cO2CnI@R(mwQ}%L{CJaP!_@#UsaVMV7b~uj!Ms6s{Iqx~8FuCwNCk`~BA(F^ zruhR8dxb=mawla;)R69z%UpNXn*=m@XAI0`y|?3w$p&!s-ZeC(ri>Zt20ZtOus~ZP z=m_*w3ICEx87*#4_QoDpR7c5WCy%6lWFd1=e46VKgOGa&9#q0hVaB)sW}2E)@y1s* z+xR|!64Age?>~%*M@3^6;;%@u73$O1%?dp5<>oVl*(jI)aJ;U-(rLMF6@INnCON9x z+oy2A;321n^avtc;F73fYN~25XRJ!rEHH81z_R{t0PS;_8z@|P;Y!7Mu0m@jyA&04 zvC8K?q+FT{Ne0*e<{}GgR%bW;4I>mB>3?SxO5BBG84ThezNf*x`5`VFI`55ubN#iQ zE!?ygekHyVz>L)zH_Y81>)l}0v1M^uMnZd+T4P074LW&WJr0e0577e2!KEZyut8r3 z>9k(nN!3YB7YcLbARROA(Z%j3O7@ z-km;e;a;oXauuxV<^|V++FBTXs65FE05R-MeCYdfu6;~zo@Vv+HxED-O z$=$QEW=UOQa-#ykns)!K6mzY%Z#L4Fg;NP1xgdGwK*Yc=I7m2iI@75<*;1pKfFe7u zEK-;}u_YF}2xR8hR*r1V3E?PWj4i2YhGKzE_L^@n4A*l$reoI9edfMcDIr*iHdR0X z4;>6>C9z7&HIOFd*KB)96u@)v3_Twuderdca(wa_wtf4I z8*C2T`(YlWPaF2GYUf{lpuT1_t|i=TYgPi==dHQuKh48m@Y8;n{QLjb6gL(CS;c4qE$8(8TeD<7Z7DJ$_930Q?5Z0?i1F zZUx#Db<4R?R?|TaDNW08RQSH!c{ipCIfMe~;6_W}e!H%DFJl2^cn2gXM{HNKuiU%I zapqgf0!Q@0(3mSxl0^HaOsqh)=AmF@nnBTO3dw^r6;XR&gY4?(5rf&?0Tfh**mjo4y6kfuaN%Cw`1<10^p@2|*wLRKSZ52S`6p?IG$F_zPNb9v9%76gRCw53Z-%6(5oqX7dpo4O z6WNWR>jZBWTi3y-7h4bqlg!DIaGQEJ*sDEVs_~!#3}@Ye3@d71MHOk)l-tKp_s=5Ti~chREt`OInRN93WR7CqUbCi=vXM4Oo)p zx#~-KqJvTX2^&3R8{0g4&nn(LK zuCN_Oc8&am!;cqa%FWpu3fBbb(Ynsm@@dID&;?jvXrHRYuz1GU1LoO-jy=a&5 z%@dD+8_$`i(vH3lT)8!osNmxH?T3X`)o!=S>#koGHGf;#4?;jMcFtM?0pohpvo@|+ zuP>MDOg%-#gXgs<#1#1xuE;^3Ant;MHS1N%NT+-n8ue*Crz@y>Kv7Nzd7M4Dk(3N~*=Ro@9SZ?WT5nlXIF zmH&E~q)M#ir3;!(KX5f*2F3XCP?(kIpOJ3K3#~@IqWVU zu)>{TeY3ctcIb&~^m!Nb^3MYc!lvqRLGzaP_a~dQ3Y#OwT28EXcau-tnn@KLS3O7j zjVD}w6&)m@*U$I-2&a=}ad4W#yYIsnMss*n#K!x|H}zMw)m>=$~Dn_%nsd@9ijcIi7sd4o+3z3?04Kk09xIfOG zwnMc?QB&h`WWQT8_TS}wu6oN4b@mEBcgnCmf8P99-)Xq#-kwi^L+MlCN{p_)62F+m zAVq2Qknukm%+ipcZ}CdgH=h*M^26El)fz|!db>4Ai7S=iKf#m?B%;8Q%dD%(jawc7 zS=kZ)tbX2o?)YUFJ9F^lnAOkkO$WGxjd;?ECo-68d1jM7>N|Hi#(y8zhtZGUm)iE2 z=zBIJd!!?ep}YF{=X5l^T7?8(JIQ9*4zK*?JDqe3#A*v?>mNbjZIPtBJ6h}HGw2W2 zomZ48G5=_*D_A%L-gjWbOSe>}L`7yGtptWmUUL5gI>Ro2l{(#_Q=vfn`*KH9>3(PP z=)5TTf!h&I2=WTvMF)Rl?cS(yX(EuXr8-#{_g&%j@zn`zT(ZkwHi$>42*}b)VDHP5 z#9G4{-^MC^H8Ua@wnlXqTIa8xII^=3+>nWD)~gq>pda(m$5%fdK4E;V=mea;l@qEz ztG)BVvCqwAJ}?`JR$0F%@C8W0@H_3D+Wtrrr~%+S;48Rj0X#}#1|&vK?W}#%Cz^^= z{t=F)n5)y!6`K1#&@-qzWl)mDz^j?Xv;}}cv3qOO*g=0>X4c?*X6Z|%Z{OesSwYv! zm&L3NF|?9svU_%m+E|$slDWw=XgVc7(5Jd@o8pOG+np&qojO2iz}}~lgs+2B%*_Ye z;JToi-BxYl)lU&ip0?+6`g-uAXs}piMiQTM&y|I42fr77jgiC-q1S$&c~2IKvQKV0 zTu%(CT^;}2L2k^1mwy#QLD8l)h*WXIapX`R5)U8%tt`_gF7uPK5 zAMDXd_b}2g!uktb6LZCjD27Jxyw!J!6 zR`g!x9&^QR!8!S6L>%oY?P^^)f)XHEHbOpcL|1HMo4YVhytK?0*kjwvX7v+smf<&T zu8T0e?(o)xz)}irH3lYohghRyd2y2n(OwG5wlC7{-u84-LAn9Sszpz-yiuBL_mgN` zW)bBA$t~LP@0lB9$BK(Z615{ESGHxtJ9*c=Z`RnyZ)?Rog8A>ZkM7JK(ggx2k>J4* zT$#7Q0zBh8kJ#05DbgE#YpO0n<5xi^->niBsFprLcmN~4BGs;2E~2603CPThU$;v? zMeh5Oap~VKn*&YUEa`#=7v7){#cXr-VrxbC-NiF<`nRjNd5U-qK2n>Z#B8V%OWfhN z9-dTt5~Xy1F8X`~jh8oltOc#oTuWd2oHD(PI}b8{}#TS7>~Bq~=dX z#*emTgCfnmOW((>r(yq=k=>fO3i2Pq6*uuQWaJBwX!y1q44;YF3HIdp=*24q-I#Ec z*d5qXsDH5>_VUI9F%XgXeX8|Pz`sG=akh6Lp?QAskkLMoPKoY4iPYzM&MI34)+2Sn z$7ssymGQF)$>fBjXS`gS)+6jh$?`LAvh&z7@S%bHA%)xB^-LH5oniTuW=e*E)>U|M zE0PE}pc#Tx;6XtXRTx)<7MN!4=Mhht-FvIn(dM}<$6N*U7{-pnll{i3+CM%f+#y?dOeP$R%TIoRh@UcD0ezC(+s& zCX9T3)GQKJd49POSE1-J2lf(IjZleT#nhACgk{$!48)jV++-)t10i0zPgbH+gyHRd z4K|tX^yex4Yg$hNa6rMiUE`HWeePG5CQ*_$&VHMOry5Z})3_+&R%!kY&lBDeI)|6B zXH;cYk{D61z72|>U}u2G^o#lCO;yPS#AADgTLUH~rd0d^?#TDnpAIJFUxEMc5+@h6 z(Om*!!{E_Rnvg2H!tP2IXgXR`s@werS!VxorC5*z z$T`gP2+%FuRC~@aPh(=RcoCu$HFnh%wdC1W;yBYA0|1-70i!%PUSd!oaEE!2o?sd$ z)8KsMbtpnXc>Lb_o=@s;YLPumHuo8bQFMZ&nosea-CUI~7bDqjdp?(rfPm`nTRt2A zq}S@;NcnhMdx}?$UlW-zw)2_fjJp=GDaf1b5J&NEq|BFl=bouYThdpHQ@x6%J3c0I zB`O4~P|0>^f79iDRck%+U8vc;fY0H9#QFBM43roCAm^jx(@J0&;u_O(`&6!*Tm}OW zWssb}vTYa4{mb0Fcdlrw{!?xmV(Zr*)+oOBhq z$hZ-f?_WP-u6=>?JH#4Iw1I5Ea9n|3U1inYK=D@H38I!ewinW}G{FW`%bA*waw*$F zYaSxKFc(mPC5^Kyg_l%1Q$$LxXdgPeUjR3q7g)7DNiTRMt*$1o{!J-LK2?!I8%<}2 zLkSL3d;e~GMAJi$_0vFcYA&XM_q#x!N z&2xn|l@G$-^7ukPT^M2cS?sr=duRX+_@(GehHZTsFb)y{oiHz6>3F)goKO=DOn$0(ZS9+X@2N{je7W9+#U%O~Ssp+?`G@!xo|*6z8gTa2J%W zz|8EA4&25NaTyxw9fa@fh2G1=(ahc_BLe(AqdZBq{2fWBoTuzcU?7KrADx3q|e z2&NZYY1IpvyXk+2;{pvWjzqOcBP?YvoL<5ChE@ zsT`q&dd9#TEsT(j<(V4J6mG$;^oAb)PQ;!Y8A)l9P}05*pK^Gh{G8zpNlO)s{LR%}>M3!0&;IMTf)zgq(zrzxT4CBV*_b(GfQ?rKb^+ z#1rk&^e>h>u0-J;uO@S!|FP1M5YpY^%KZ8jfVjh(q|iShn>y?)8hpHwcE3=Cd;`hD z9jhhtg4WP7y&#gYkz<9I@-Okj&Mh?Qi>G!L^}AMs@%$#(6mm2qZ`R55wD=-x=a;l#&W}1@m#-1w z$~|Z{;fAKro%m^}L_KFdCD!Hi00i~i-T_9;oh2ob%71D2;FD|)kg!Sq=G%t)RU(NE z1nUd`rX({S#O9lc>7Y8W!FXJ&SKVpN*pUQ^wPN*X#mCjePalML2tTOSkeCw-%c{aBA<@7O8jBw9b9=} z0FIYKMBHh%GdZkR`(~rh3pp5Q_-~^`OC9sZqJn1vGSwUyKUQzdf#lo!U`ve~PKnoY z)>VJ#zZARNd@i0yUs7`#vf6w9Ywz0sne6+xRCkf?+B&J+rP+$L(lbLjG-_g-Lnut; zP#cvwr8{Yl)s0)0oYv%CQgTRca;%ush#rTmX%*&>CqkryjAG*XTz+`|iRZa~+iS1a z^}4Rt=X-rWpYQwq{=7f&IMI%@^L%qdAAH{ehI2HCud7SSyTCI&7 zA*Lo?1Kk@6yMYM$>6CN(t+j%qlJh6jJctrqz)oQF0Fi+ShIzWGmM**GDTKJ%Kt#?ZR(t-NEnnJ_S>z#akB9rRaVB2lTRmRCS7b-z)J**fP z7VOdM3U8!sNZJcK`yc6Mj{&a%|FhQXbIRi!!H>aKs;NuVp30`c6&|TR4D6*sUHaGW zXtjfY!u5hEBrp;)Juh5yQ;HeoHwj8eDa%w_PRm=Ef2vixw)$6mYuOo6ZJQJU3+oTB zD_UX=1OL$%kF9xf17>Ka=irVfRjWR{d}jW=&EJTT3dVEY81F71e0qT|>+KPR_`5Md zc+@jHJrU0NN&Y$FPynq`F8$A4wj+2c$T?(Ee#13YoLVcW=vk~{ARoUd#FA{q6EI6K zMZN#tioKb_x5B$}jYB2X`VUeLV6Jkrp?d8g-bTEhEpwPb6VIgDI5sUDazN|osnGV; zYVZ1AzC&4lwlK9J9`0=mm`DCQp|h#!4gd5bB*Y-kI`wC5vj_;=+U@H?B#WctFRa7o zZJ=f#GR=0^2IAQZ_fclmfS8AgcQgJ*c(q?`n5TT6&+3VP8hZ!3n1OSg$fE^Q(;)lL06D$jk91I#+G!$W2y98!MI& zx81oe1HA?N#z<*eyr8g ziQ}s*a;te&w8QQelYzZ+K^RMs+hJd9BAxciE{FT|*f&l>qX(ow!<-JhJ3g(sbLC8C zlRAebINd)qss8w~V}rv?L2_-ej0qk!tUE87A1RDDQ*2}Cp;taj%>$vxF=2#Hh{x5E z%?@oCYVAKW5W5^WzH@GRXOJ58MC_#O{^7`PMvR`4waGKI6@nk4VfW>4DgBTmmINSO zK{|D;IQ6D1g$qO}Nk#O+_7=>~0ow;5X5ndhh$d(eTBuc0INJyD_nq7Xu`MBk4q(@W zc;~IkBkm({UU^_M>OQGcoNYl>kJx9ewQ{Np?3hO4?(sncov@TH?>~JG5p1t>J^l`v zR>3Mmd_{knV1l34FKZlUL_UqIgBfnZqmf3Mb(UY=LQn^%`@XKO=KxL^Af6t+jZNDd zoB>o~J4DVhagIFtAd|iH;F$~Sb%W7yDpw!t%BS`fy^SIuM+mmnVwQ0d330V#FQoxR z!rraP8ivhycV+@2j(aF*?9|}e(+7UrcrBkehY3cWJ=$@Rhg~mga)CV077 zAisBO5d&psWRXy^+4-uKEat9$NpU`kQGD`GiNPf}^bu#=?1d#0L1Uf8*>g%+drFXd z{M}I24eV!l*1Dj-uOB*lhfT-lS)YOKiU5F{84}x!cc}z)_I^YYMtPs2@e+iUJ<#LY z3rZEXAKfUR$8D&wjkACR6*`?6oRAS|gpR?1Z{}%uw9!(U)1I;1Q83ZYwd;lEJRs@Z zY%2|BpAHKU#=IBk`7LdQ_`G1(D zusSe~8SvuQgfs-eRDxWPJ0Y!B#SPUy3g|V-%yDYpn`njcd$#h57nM@=2JBiTz;$lH zHxZJ;28SD0?C5m+a!~3Oj5Ap11pTSuG_{@w!a=9QNX57EX=Zb2x$KevIj1RcFK(P+ z=q^pn?N2ewT&!C;XtLh|8+Y-6_7Cs9I62NIHaYsTwHO@yp5PYaRPn@0sCm^C6;|qS z#$KysCnq!!YBt?tB7|(#f^N+ub7S&-7>nD{wT9sE48u47)uiZb)Uq|)sSkPt|2Ax% zou>Blm^jQcdLSvB3`oZXnmtwxi3);AOevc69$|3wKx7o`Z^435TK!8OAI!_^H0csn z84wTTuYMb-Cfxgfi5jFj8?i%o*EOmzZVpY2zVT~JZ|(=n_mF(elmE`PK|{$Ip%X%x zsk!JJI&Q)9l))ERINXZ0A4{~FfqpLn@-0zj>uFn~ZJjW_W&QhxZx28wFCFN6#c7Ca z1#$A&Wl;5k&qw4^+9ow^Dg@;~9dGlKRDF-C&bd&>1r~aHm_d3;IV3cBx*1Isn9QP* z&5{Qg^mSmjIwR?ziTadId8vitx_}GNJD8L-3a#LxOvRxftAFJJEA!5yJSG8NU)Rm* zK&`XjD1Ov2BXk7t)w2CD{ch3p^*qZkL{G=F`TB$L8<|^W_eKou7d;z&Z+pvo5ZM?v zTCZ*$vOTris8HeLf!=hh0^nueJgen`zp^A2rV%-C{2+{HIq){_p5@;kbIGw!ODsH& z4eu%o43y(;)r6gXcQ3kF)vO^AH8gw<$a|B++jyDqwy*LT2FUSz!7N;$iEsXJ1$4Y} zv2WN8aUzI7&gQU@C%Xt$w7xM=W#6F`?snHEQU!I>ec*tWJGmRyi{9Atwc|%YVrC(W zJJqBxNjwThYj+j~!^AdxvzPrO#mY*vj?~zkZ;N~c8EP`%(0A*6Opr{PPfV9T zs`9;E@VoEU%5bGaKwXREbI%B;{KWjJ6dO-J#$^O3&xOeu_K!=O&wUzO<}K#$|YCUeF2 z^NKj4HgcheOCaYlBlT_=7+GMc5LPVG6j}}SLsEt4#Re?@NZK?3lvbQJjdoL(GT?=Ld3LqteurVekijwn| z0vCWWXcP)H4h0naLTDiPg<7R9_#giBAhO0H%DmpJHy)^-^!7Y7R#o%f)HR^v#M2rL z6iO{gH9M-itR>)cR7G-chO`U_sbc5HC-;SVP0nj5i4+Q!0{W+dmy6bYN{ESxVLF)8 zFmP=VIHFN%^3pH4~u4$`h}A`Zgi{GaQB7!!c{i{{!^jRXV>abil?>-2NtC1um+UJ*D7)SHyn+BVi>L From 5ef1040c39f56ee12c34352ad677688d5ffb394b Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Wed, 16 Aug 2023 14:06:42 +0800 Subject: [PATCH 174/337] Delete README.md --- README.md | 759 ------------------------------------------------------ 1 file changed, 759 deletions(-) delete mode 100644 README.md diff --git a/README.md b/README.md deleted file mode 100644 index 9de1c43c..00000000 --- a/README.md +++ /dev/null @@ -1,759 +0,0 @@ -# Non-parallel training: -## DL: -### Save and restore: -```python -import Note.DL.kernel as k #import kernel module -import tensorflow as tf #import tensorflow library -import nn as n #import neural network module -mnist=tf.keras.datasets.mnist #load mnist dataset -(x_train,y_train),(x_test,y_test)=mnist.load_data() #split data into train and test sets -x_train,x_test =x_train/255.0,x_test/255.0 #normalize data -nn=n.nn() #create neural network object -kernel=k.kernel(nn) #create kernel object with the network -kernel.platform=tf #set the platform to tensorflow -kernel.data(x_train,y_train) #input train data to the kernel -kernel.train(32,5) #train the network with batch size 32 and epoch 5 -kernel.save() #save the neural network to a file -``` -```python -import Note.DL.kernel as k #import kernel module -import tensorflow as tf #import tensorflow library -mnist=tf.keras.datasets.mnist #load mnist dataset -(x_train,y_train),(x_test,y_test)=mnist.load_data() #split data into train and test sets -x_train,x_test =x_train/255.0,x_test/255.0 #normalize data -kernel=k.kernel() #create kernel object without a network -kernel.platform=tf #set the platform to tensorflow -kernel.data(x_train,y_train) #input train data to the kernel -kernel.restore('save.dat') #restore the network from a file -kernel.train(32,1) #train the network again with batch size 32 and epoch 1 -``` - -## Training with test data -**You can get a neural network example from the link below, and then you can import neural network and train with kernel, example code are below.** - -https://github.com/NoteDancing/Note-documentation/blob/Note-7.0/Note%207.0%20documentation/DL/neural%20network/tensorflow/nn_acc.py - -```python -import Note.DL.kernel as k #import kernel module -import tensorflow as tf #import tensorflow library -import nn_acc as n #import neural network module with accuracy function -mnist=tf.keras.datasets.mnist #load mnist dataset -(x_train,y_train),(x_test,y_test)=mnist.load_data() #split data into train and test sets -x_train,x_test =x_train/255.0,x_test/255.0 #normalize data -nn=n.nn() #create neural network object -kernel=k.kernel(nn) #create kernel object with the network -kernel.platform=tf #set the platform to tensorflow -kernel.data(x_train,y_train,x_test,y_test) #input train and test data and labels to the kernel -kernel.train(32,5,32) #train the network with batch size 32, epoch 5 and test batch size 32 -kernel.test(x_test,y_test,32)#test the network performance on the test set with batch size 32 -``` - -### Saving multiple files in training: -```python -import Note.DL.kernel as k #import kernel module -import tensorflow as tf #import tensorflow library -import nn as n #import neural network module -mnist=tf.keras.datasets.mnist #load mnist dataset -(x_train,y_train),(x_test,y_test)=mnist.load_data() #split data into train and test sets -x_train,x_test =x_train/255.0,x_test/255.0 #normalize data -nn=n.nn() #create neural network object -kernel=k.kernel(nn) #create kernel object with the network -kernel.platform=tf #set the platform to tensorflow -kernel.data(x_train,y_train) #input train data to the kernel -kernel.train(32,5,save=True,one=False,s=3) #train the network with batch size 32 and epoch 5 -``` - -### Stop training and saving when condition is met: -```python -import Note.DL.kernel as k #import kernel module -import tensorflow as tf #import tensorflow library -import nn as n #import neural network module -mnist=tf.keras.datasets.mnist #load mnist dataset -(x_train,y_train),(x_test,y_test)=mnist.load_data() #split data into train and test sets -x_train,x_test =x_train/255.0,x_test/255.0 #normalize data -nn=n.nn() #create neural network object -kernel=k.kernel(nn) #create kernel object with the network -kernel.platform=tf #set the platform to tensorflow -kernel.stop=True #set the flag to stop training when a condition is met -kernel.end_loss=0.7 #set the condition to stop training when the loss is less than 0.7 -kernel.data(x_train,y_train) #input train data to the kernel -kernel.train(32,5) #train the network with batch size 32 and epoch 5 -``` - -### Visualization: -```python -import Note.DL.kernel as k #import kernel module -import tensorflow as tf #import tensorflow library -import nn as n #import neural network module -mnist=tf.keras.datasets.mnist #load mnist dataset -(x_train,y_train),(x_test,y_test)=mnist.load_data() #split data into train and test sets -x_train,x_test =x_train/255.0,x_test/255.0 #normalize data -nn=n.nn() #create neural network object -kernel=k.kernel(nn) #create kernel object with the network -kernel.platform=tf #set the platform to tensorflow -kernel.stop=True #set the flag to stop training when a condition is met -kernel.end_loss=0.7 #set the condition to stop training when the loss is less than 0.7 -kernel.data(x_train,y_train) #input train data to the kernel -kernel.train(32,5) #train the network with batch size 32 and epoch 5 -kernel.visualize_train() #visualize the loss -``` - -### Set the print count: -```python -import Note.DL.kernel as k #import kernel module -import tensorflow as tf #import tensorflow library -import nn as n #import neural network module -mnist=tf.keras.datasets.mnist #load mnist dataset -(x_train,y_train),(x_test,y_test)=mnist.load_data() #split data into train and test sets -x_train,x_test =x_train/255.0,x_test/255.0 #normalize data -nn=n.nn() #create neural network object -kernel=k.kernel(nn) #create kernel object with the network -kernel.platform=tf #set the platform to tensorflow -kernel.data(x_train,y_train) #input train data to the kernel -kernel.train(32,5,p=3) #train the network with batch size 32 and epoch 5 -``` - - -## RL: -### Saving multiple files in training: -```python -import Note.RL.kernel as k #import kernel module -import tensorflow as tf #import tensorflow library -import DQN as d #import deep Q-network module -dqn=d.DQN(4,128,2) #create neural network object with 4 inputs, 128 hidden units and 2 outputs -kernel=k.kernel(dqn) #create kernel object with the network -kernel.platform=tf #set the platform to tensorflow -kernel.action_count=2 #set the number of actions to 2 -kernel.set_up(epsilon=0.01,pool_size=10000,batch=64,update_step=10) #set up the hyperparameters for training -kernel.train(100,save=True,one=False,s=3) #train the network for 100 episodes -``` - -### Stop training and saving when condition is met: -```python -import Note.RL.kernel as k #import kernel module -import DQN as d #import deep Q-network module -dqn=d.DQN(4,128,2) #create neural network object with 4 inputs, 128 hidden units and 2 outputs -kernel=k.kernel(dqn) #create kernel object with the network -kernel.stop=True #set the flag to stop training when a condition is met -kernel.action_count=2 #set the number of actions to 2 -kernel.set_up(epsilon=0.01,pool_size=10000,batch=64,update_step=10,trial_count=10,criterion=200) #set up the hyperparameters for training and the condition to stop training when the average reward of 10 trials is greater than 200 -kernel.train(100) #train the network for 500 episodes -``` - -### Visualization: -```python -import Note.RL.kernel as k #import kernel module -import DQN as d #import deep Q-network module -dqn=d.DQN(4,128,2) #create neural network object with 4 inputs, 128 hidden units and 2 outputs -kernel=k.kernel(dqn) #create kernel object with the network -kernel.stop=True #set the flag to stop training when a condition is met -kernel.action_count=2 #set the number of actions to 2 -kernel.set_up(epsilon=0.01,pool_size=10000,batch=64,update_step=10,trial_count=10,criterion=200) #set up the hyperparameters for training and the condition to stop training when the average reward of 10 trials is greater than 200 -kernel.train(100) #train the network for 500 episodes -kernel.visualize_reward() #visualize the reward -kernel.visualize_train() #visualize the loss -``` - -### Set the print count: -```python -import Note.RL.kernel as k #import kernel module -import DQN as d #import deep Q-network module -dqn=d.DQN(4,128,2) #create neural network object with 4 inputs, 128 hidden units and 2 outputs -kernel=k.kernel(dqn) #create kernel object with the network -kernel.stop=True #set the flag to stop training when a condition is met -kernel.action_count=2 #set the number of actions to 2 -kernel.set_up(epsilon=0.01,pool_size=10000,batch=64,update_step=10,trial_count=10,criterion=200) #set up the hyperparameters for training and the condition to stop training when the average reward of 10 trials is greater than 200 -kernel.train(100,p=3) #train the network for 500 episodes -``` - - -## Parallel test: -**You can get a neural network example from the link below, and then you can import neural network and train with kernel, example code are below.** - -https://github.com/NoteDancing/Note-documentation/blob/Note-7.0/Note%207.0%20documentation/DL/neural%20network/tensorflow/parallel/nn.py - -```python -import Note.DL.kernel as k #import kernel module -import tensorflow as tf #import tensorflow library -import nn as n #import neural network module -mnist=tf.keras.datasets.mnist #load mnist dataset -(x_train,y_train),(x_test,y_test)=mnist.load_data() #split data into train and test sets -x_train,x_test =x_train/255.0,x_test/255.0 #normalize data -nn=n.nn() #create neural network object -nn.build() #build the network structure -kernel=k.kernel(nn) #create kernel object with the network -kernel.platform=tf #set the platform to tensorflow -kernel.process_t=3 #set the number of processes to test -kernel.data(x_train,y_train,x_test,y_test) #input train and test data to the kernel -kernel.train(32,5,32) #train the network with batch size 32, epoch 5 and test batch size 32 -``` - - -# Parallel training: -## DL: -### PO1: -```python -import Note.DL.parallel.kernel as k #import kernel module -import tensorflow as tf #import tensorflow library -import nn as n #import neural network module -from multiprocessing import Process,Lock,Manager #import multiprocessing tools -mnist=tf.keras.datasets.mnist #load mnist dataset -(x_train,y_train),(x_test,y_test)=mnist.load_data() #split data into train and test sets -x_train,x_test =x_train/255.0,x_test/255.0 #normalize data -nn=n.nn() #create neural network object -nn.build() #build the network structure -kernel=k.kernel(nn) #create kernel object with the network -kernel.process=3 #set the number of processes to train -kernel.epoch=5 #set the number of epochs to train -kernel.batch=32 #set the batch size -kernel.PO=1 #use PO1 algorithm for parallel optimization -kernel.data(x_train,y_train) #input train data to the kernel -manager=Manager() #create manager object to share data among processes -kernel.init(manager) #initialize shared data with the manager -lock=[Lock(),Lock()] #create two locks for synchronization -for p in range(3): #loop over the processes - Process(target=kernel.train,args=(p,lock)).start() #start each process with the train function and pass the process id and locks as arguments -kernel.update_nn_param() #update the network parameters after training -kernel.test(x_train,y_train,32) #test the network performance on the train set with batch size 32 -``` - -### PO2: -```python -import Note.DL.parallel.kernel as k #import kernel module -import tensorflow as tf #import tensorflow library -import nn as n #import neural network module -from multiprocessing import Process,Lock,Manager #import multiprocessing tools -mnist=tf.keras.datasets.mnist #load mnist dataset -(x_train,y_train),(x_test,y_test)=mnist.load_data() #split data into train and test sets -x_train,x_test =x_train/255.0,x_test/255.0 #normalize data -x_train=x_train.reshape([60000,784]) #reshape data to fit the network input -nn=n.nn() #create neural network object -nn.build() #build the network structure -kernel=k.kernel(nn) #create kernel object with the network -kernel.process=3 #set the number of processes to train -kernel.epoch=5 #set the number of epochs to train -kernel.batch=32 #set the batch size -kernel.PO=2 #use PO2 algorithm for parallel optimization -kernel.data(x_train,y_train) #input train data to the kernel -manager=Manager() #create manager object to share data among processes -kernel.init(manager) #initialize shared data with the manager -lock=[Lock(),Lock()] #create two locks for synchronization -g_lock=Lock() #create a global lock for gradient computing -for p in range(3): #loop over the processes - Process(target=kernel.train,args=(p,lock,g_lock)).start() #start each process with the train function and pass the process id and locks as arguments -kernel.update_nn_param() #update the network parameters after training -kernel.test(x_train,y_train,32) #test the network performance on the train set with batch size 32 -``` -```python -import Note.DL.parallel.kernel as k #import kernel module -import tensorflow as tf #import tensorflow library -import nn as n #import neural network module -from multiprocessing import Process,Lock,Manager #import multiprocessing tools -mnist=tf.keras.datasets.mnist #load mnist dataset -(x_train,y_train),(x_test,y_test)=mnist.load_data() #split data into train and test sets -x_train,x_test =x_train/255.0,x_test/255.0 #normalize data -x_train=x_train.reshape([60000,784]) #reshape data to fit the network input -nn=n.nn() #create neural network object -nn.build() #build the network structure -kernel=k.kernel(nn) #create kernel object with the network -kernel.process=3 #set the number of processes to train -kernel.epoch=5 #set the number of epochs to train -kernel.batch=32 #set the batch size -kernel.PO=2 #use PO2 algorithm for parallel optimization -kernel.data(x_train,y_train) #input train data to the kernel -manager=Manager() #create manager object to share data among processes -kernel.init(manager) #initialize shared data with the manager -lock=[Lock(),Lock()] #create two locks for synchronization -g_lock=[Lock(),Lock()] #create a list of global locks for gradient computing -for p in range(3): #loop over the processes - Process(target=kernel.train,args=(p,lock,g_lock)).start() #start each process with the train function and pass the process id, the locks and the global locks as arguments -kernel.update_nn_param() #update the network parameters after training -kernel.test(x_train,y_train,32) #test the network performance on the train set with batch size 32 -``` - -### PO3： -```python -import Note.DL.parallel.kernel as k #import kernel module -import tensorflow as tf #import tensorflow library -import nn as n #import neural network module -from multiprocessing import Process,Lock,Manager #import multiprocessing tools -mnist=tf.keras.datasets.mnist #load mnist dataset -(x_train,y_train),(x_test,y_test)=mnist.load_data() #split data into train and test sets -x_train,x_test =x_train/255.0,x_test/255.0 #normalize data -nn=n.nn() #create neural network object -nn.build() #build the network structure -kernel=k.kernel(nn) #create kernel object with the network -kernel.process=3 #set the number of processes to train -kernel.epoch=5 #set the number of epochs to train -kernel.batch=32 #set the batch size -kernel.PO=3 #use PO3 algorithm for parallel optimization -kernel.data(x_train,y_train) #input train data to the kernel -manager=Manager() #create manager object to share data among processes -kernel.init(manager) #initialize shared data with the manager -lock=Lock() #create a lock for synchronization -for p in range(3): #loop over the processes - Process(target=kernel.train,args=(p,lock)).start() #start each process with the train function and pass the process id and the lock as arguments -kernel.update_nn_param() #update the network parameters after training -kernel.test(x_train,y_train,32) #test the network performance on the train set with batch size 32 -``` - -### Save and restore: -```python -import Note.DL.parallel.kernel as k #import kernel module -import tensorflow as tf #import tensorflow library -import nn as n #import neural network module -from multiprocessing import Process,Manager #import multiprocessing tools -mnist=tf.keras.datasets.mnist #load mnist dataset -(x_train,y_train),(x_test,y_test)=mnist.load_data() #split data into train and test sets -x_train,x_test =x_train/255.0,x_test/255.0 #normalize data -nn=n.nn() #create neural network object -nn.build() #build the network structure -kernel=k.kernel(nn) #create kernel object with the network -kernel.process=3 #set the number of processes to train -kernel.epoch=5 #set the number of epochs to train -kernel.batch=32 #set the batch size -kernel.PO=3 #use PO3 algorithm for parallel optimization -kernel.data(x_train,y_train) #input train data to the kernel -manager=Manager() #create manager object to share data among processes -kernel.init(manager) #initialize shared data with the manager -for p in range(3): #loop over the processes - Process(target=kernel.train,args=(p,)).start() #start each process with the train function and pass the process id as argument -kernel.update_nn_param() #update the network parameters after training -kernel.test(x_train,y_train,32) #test the network performance on the train set with batch size 32 -kernel.save() #save the neural network to a file -``` -```python -import Note.DL.parallel.kernel as k #import kernel module -import tensorflow as tf #import tensorflow library -import nn as n #import neural network module -from multiprocessing import Process,Manager #import multiprocessing tools -mnist=tf.keras.datasets.mnist #load mnist dataset -(x_train,y_train),(x_test,y_test)=mnist.load_data() #split data into train and test sets -x_train,x_test =x_train/255.0,x_test/255.0 #normalize data -kernel=k.kernel() #create kernel object without a network -kernel.process=3 #set the number of processes to train -kernel.epoch=1 #set the number of epochs to train -kernel.batch=32 #set the batch size -kernel.PO=3 #use PO3 algorithm for parallel optimization -kernel.data(x_train,y_train) #input train data to the kernel -manager=Manager() #create manager object to share data among processes -kernel.init(manager) #initialize shared data with the manager -kernel.restore('save.dat') #restore the neural network from a file -for p in range(3): #loop over the processes - Process(target=kernel.train,args=(p,)).start() #start each process with the train function and pass the process id as argument -``` - -### Visualization: -```python -import Note.DL.parallel.kernel as k #import kernel module -import tensorflow as tf #import tensorflow library -import nn as n #import neural network module -from multiprocessing import Process,Lock,Manager #import multiprocessing tools -mnist=tf.keras.datasets.mnist #load mnist dataset -(x_train,y_train),(x_test,y_test)=mnist.load_data() #split data into train and test sets -x_train,x_test =x_train/255.0,x_test/255.0 #normalize data -nn=n.nn() #create neural network object -nn.build() #build the network structure -kernel=k.kernel(nn) #create kernel object with the network -kernel.process=3 #set the number of processes to train -kernel.epoch=5 #set the number of epochs to train -kernel.batch=32 #set the batch size -kernel.PO=3 #use PO3 algorithm for parallel optimization -kernel.data(x_train,y_train) #input train data to the kernel -manager=Manager() #create manager object to share data among processes -kernel.init(manager) #initialize shared data with the manager -lock=Lock() #create a lock for synchronization -for p in range(3): #loop over the processes - Process(target=kernel.train,args=(p,lock)).start() #start each process with the train function and pass the process id and the lock as arguments -kernel.update_nn_param() #update the network parameters after training -kernel.test(x_train,y_train,32) #test the network performance on the train set with batch size 32 -kernel.visualize_train() #visualize the loss -``` - -### Parallel test: -```python -import Note.DL.parallel.kernel as k #import kernel module -import tensorflow as tf #import tensorflow library -import nn as n #import neural network module -from multiprocessing import Process,Manager #import multiprocessing tools -mnist=tf.keras.datasets.mnist #load mnist dataset -(x_train,y_train),(x_test,y_test)=mnist.load_data() #split data into train and test sets -x_train,x_test =x_train/255.0,x_test/255.0 #normalize data -nn=n.nn() #create neural network object -nn.build() #build the network structure -kernel=k.kernel(nn) #create kernel object with the network -kernel.process=3 #set the number of processes to train -kernel.process_t=3 #set the number of processes to test -kernel.epoch=5 #set the number of epochs to train -kernel.batch=32 #set the batch size -kernel.PO=3 #use PO3 algorithm for parallel optimization -kernel.data(x_train,y_train,x_test,y_test) #input train and test data to the kernel -manager=Manager() #create manager object to share data among processes -kernel.init(manager) #initialize shared data with the manager -for p in range(3): #loop over the processes - Process(target=kernel.train,args=(p,None,None,32)).start() #start each process with the train function and pass the process id, the locks, the test flag and the test batch size as arguments -``` - -### Saving multiple files in parallel training: -```python -import Note.DL.parallel.kernel as k #import kernel module -import tensorflow as tf #import tensorflow library -import nn as n #import neural network module -from multiprocessing import Process,Manager #import multiprocessing tools -mnist=tf.keras.datasets.mnist #load mnist dataset -(x_train,y_train),(x_test,y_test)=mnist.load_data() #split data into train and test sets -x_train,x_test =x_train/255.0,x_test/255.0 #normalize data -nn=n.nn() #create neural network object -nn.build() #build the network structure -kernel=k.kernel(nn) #create kernel object with the network -kernel.process=3 #set the number of processes to train -kernel.epoch=5 #set the number of epochs to train -kernel.batch=32 #set the batch size -kernel.PO=3 #use PO3 algorithm for parallel optimization -kernel.s=3 #set the maximum number of saved files -kernel.data(x_train,y_train) #input train data to the kernel -manager=Manager() #create manager object to share data among processes -kernel.init(manager) #initialize shared data with the manager -for p in range(3): #loop over the processes - Process(target=kernel.train,args=(p,)).start() #start each process with the train function and pass the process id and the lock as arguments -kernel.update_nn_param() #update the network parameters after training -kernel.test(x_train,y_train,32) #test the network performance on the train set with batch size 32 -``` - -### Stop training and saving when condition is met: -```python -import Note.DL.parallel.kernel as k #import kernel module -import tensorflow as tf #import tensorflow library -import nn as n #import neural network module -from multiprocessing import Process,Lock,Manager #import multiprocessing tools -mnist=tf.keras.datasets.mnist #load mnist dataset -(x_train,y_train),(x_test,y_test)=mnist.load_data() #split data into train and test sets -x_train,x_test =x_train/255.0,x_test/255.0 #normalize data -x_train=x_train.reshape([60000,784]) #reshape data to fit the network input -nn=n.nn() #create neural network object -nn.build() #build the network structure -kernel=k.kernel(nn) #create kernel object with the network -kernel.stop=True #set the flag to stop training when a condition is met -kernel.end_loss=0.7 #set the condition to stop training when the loss is less than 0.7 -kernel.process=3 #set the number of processes to train -kernel.epoch=5 #set the number of epochs to train -kernel.batch=32 #set the batch size -kernel.PO=3 #use PO3 algorithm for parallel optimization -kernel.data(x_train,y_train) #input train data to the kernel -manager=Manager() #create manager object to share data among processes -kernel.init(manager) #initialize shared data with the manager -lock=Lock() #create a lock for synchronization -for p in range(3): #loop over the processes - Process(target=kernel.train,args=(p,lock)).start() #start each process with the train function and pass the process id and locks as arguments -``` - -### Process priority: -```python -import Note.DL.parallel.kernel as k #import kernel module -import tensorflow as tf #import tensorflow library -import nn as n #import neural network module -from multiprocessing import Process,Manager #import multiprocessing tools -mnist=tf.keras.datasets.mnist #load mnist dataset -(x_train,y_train),(x_test,y_test)=mnist.load_data() #split data into train and test sets -x_train,x_test =x_train/255.0,x_test/255.0 #normalize data -nn=n.nn() #create neural network object -nn.build() #build the network structure -kernel=k.kernel(nn) #create kernel object with the network -kernel.process=3 #set the number of processes to train -kernel.epoch=5 #set the number of epochs to train -kernel.batch=32 #set the batch size -kernel.priority_flag=True #set the flag to use priority scheduling for processes -kernel.PO=3 #use PO3 algorithm for parallel optimization -kernel.data(x_train,y_train) #input train data to the kernel -manager=Manager() #create manager object to share data among processes -kernel.init(manager) #initialize shared data with the manager -for p in range(3): #loop over the processes - Process(target=kernel.train,args=(p,)).start() #start each process with the train function and pass the process id and locks as arguments -kernel.update_nn_param() #update the network parameters after training -kernel.test(x_train,y_train,32) #test the network performance on the train set with batch size 32 -``` - -### Gradient attenuation: -**Calculate the attenuation coefficient based on the optimization counter using the attenuation function.** - -**You can get a neural network example from the link below, and then you can import neural network and train with kernel, example code are below.** - -https://github.com/NoteDancing/Note-documentation/blob/Note-7.0/Note%207.0%20documentation/DL/neural%20network/tensorflow/parallel/nn_attenuate.py -```python -import Note.DL.parallel.kernel as k #import kernel module -import tensorflow as tf #import tensorflow library -import nn_attenuate as n #import neural network module -from multiprocessing import Process,Manager #import multiprocessing tools -mnist=tf.keras.datasets.mnist #load mnist dataset -(x_train,y_train),(x_test,y_test)=mnist.load_data() #split data into train and test sets -x_train,x_test =x_train/255.0,x_test/255.0 #normalize data -nn=n.nn() #create neural network object -nn.build() #build the network structure -kernel=k.kernel(nn) #create kernel object with the network -kernel.process=3 #set the number of processes to train -kernel.epoch=5 #set the number of epochs to train -kernel.batch=32 #set the batch size -kernel.priority_flag=True #set the flag to use priority scheduling for processes -kernel.PO=3 #use PO3 algorithm for parallel optimization -kernel.data(x_train,y_train) #input train data to the kernel -manager=Manager() #create manager object to share data among processes -kernel.init(manager) #initialize shared data with the manager -for p in range(3): #loop over the processes - Process(target=kernel.train,args=(p,)).start() #start each process with the train function and pass the process id and locks as arguments -kernel.update_nn_param() #update the network parameters after training -kernel.test(x_train,y_train,32) #test the network performance on the train set with batch size 32 -``` - - -## RL： -**Pool Network:** -### PO1: -```python -import Note.RL.parallel.kernel as k #import kernel module -import DQN as d #import deep Q-network module -from multiprocessing import Process,Lock,Manager #import multiprocessing tools -dqn=d.DQN(4,128,2) #create neural network object with 4 inputs, 128 hidden units and 2 outputs -kernel=k.kernel(dqn,5) #create kernel object with the network and 5 processes to train -kernel.episode=100 #set the number of episodes to 100 -manager=Manager() #create manager object to share data among processes -kernel.init(manager) #initialize shared data with the manager -kernel.action_count=2 #set the number of actions to 2 -kernel.set_up(epsilon=0.01,pool_size=10000,batch=64,update_step=10) #set up the hyperparameters for training -kernel.PO=1 #use PO1 algorithm for parallel optimization -pool_lock=[Lock(),Lock(),Lock(),Lock(),Lock()] #create a list of locks for each process's replay pool -lock=[Lock(),Lock(),Lock()] #create a list of locks for synchronization -for p in range(5): #loop over the processes - Process(target=kernel.train,args=(p,lock,pool_lock)).start() #start each process with the train function and pass the process id, the number of episodes, the locks and the pool locks as arguments -``` - -### PO2: -```python -import Note.RL.parallel.kernel as k #import kernel module -import DQN as d #import deep Q-network module -from multiprocessing import Process,Lock,Manager #import multiprocessing tools -dqn=d.DQN(4,128,2) #create neural network object with 4 inputs, 128 hidden units and 2 outputs -kernel=k.kernel(dqn,5) #create kernel object with the network and 5 processes to train -kernel.episode=100 #set the number of episodes to 100 -manager=Manager() #create manager object to share data among processes -kernel.init(manager) #initialize shared data with the manager -kernel.action_count=2 #set the number of actions to 2 -kernel.set_up(epsilon=0.01,pool_size=10000,batch=64,update_step=10) #set up the hyperparameters for training -kernel.PO=2 #use PO2 algorithm for parallel optimization -pool_lock=[Lock(),Lock(),Lock(),Lock(),Lock()] #create a list of locks for each process's replay pool -lock=[Lock(),Lock(),Lock()] #create a list of locks for synchronization -g_lock=Lock() #create a global lock for gradient computing -for p in range(5): #loop over the processes - Process(target=kernel.train,args=(p,lock,pool_lock,g_lock)).start() #start each process with the train function and pass the process id, the number of episodes, the locks, the pool locks and the global lock as arguments -``` - -### PO3: -```python -import Note.RL.parallel.kernel as k #import kernel module -import DQN as d #import deep Q-network module -from multiprocessing import Process,Lock,Manager #import multiprocessing tools -dqn=d.DQN(4,128,2) #create neural network object with 4 inputs, 128 hidden units and 2 outputs -kernel=k.kernel(dqn,5) #create kernel object with the network and 5 processes to train -kernel.episode=100 #set the number of episodes to 100 -manager=Manager() #create manager object to share data among processes -kernel.init(manager) #initialize shared data with the manager -kernel.action_count=2 #set the number of actions to 2 -kernel.set_up(epsilon=0.01,pool_size=10000,batch=64,update_step=10) #set up the hyperparameters for training -kernel.PO=3 #use PO3 algorithm for parallel optimization -pool_lock=[Lock(),Lock(),Lock(),Lock(),Lock()] #create a list of locks for each process's replay pool -lock=[Lock(),Lock(),Lock()] #create three locks for synchronization -for p in range(5): #loop over the processes - Process(target=kernel.train,args=(p,lock,pool_lock)).start() #start each process with the train function and pass the process id, the number of episodes, the locks and the pool locks as arguments -``` - -### Visualization: -```python -import Note.RL.parallel.kernel as k #import kernel module -import DQN as d #import deep Q-network module -from multiprocessing import Process,Lock,Manager #import multiprocessing tools -dqn=d.DQN(4,128,2) #create neural network object with 4 inputs, 128 hidden units and 2 outputs -kernel=k.kernel(dqn,5) #create kernel object with the network and 5 processes to train -kernel.episode=100 #set the number of episodes to 100 -manager=Manager() #create manager object to share data among processes -kernel.init(manager) #initialize shared data with the manager -kernel.action_count=2 #set the number of actions to 2 -kernel.set_up(epsilon=0.01,pool_size=10000,batch=64,update_step=10) #set up the hyperparameters for training -kernel.PO=3 #use PO3 algorithm for parallel optimization -pool_lock=[Lock(),Lock(),Lock(),Lock(),Lock()] #create a list of locks for each process's replay pool -lock=[Lock(),Lock(),Lock()] #create three locks for synchronization -for p in range(5): #loop over the processes - Process(target=kernel.train,args=(p,lock,pool_lock)).start() #start each process with the train function and pass the process id, the number of episodes, the locks and the pool locks as arguments -kernel.visualize_reward() #visualize the reward -kernel.visualize_train() #visualize the loss -``` - -### Saving multiple files in parallel training: -```python -import Note.RL.parallel.kernel as k #import kernel module -import DQN as d #import deep Q-network module -from multiprocessing import Process,Lock,Manager #import multiprocessing tools -dqn=d.DQN(4,128,2) #create neural network object with 4 inputs, 128 hidden units and 2 outputs -kernel=k.kernel(dqn,5) #create kernel object with the network and 5 processes to train -kernel.episode=100 #set the number of episodes to 100 -manager=Manager() #create manager object to share data among processes -kernel.init(manager) #initialize shared data with the manager -kernel.action_count=2 #set the number of actions to 2 -kernel.set_up(epsilon=0.01,pool_size=10000,batch=64,update_step=10) #set up the hyperparameters for training -kernel.PO=3 #use PO3 algorithm for parallel optimization -kernel.s=3 #set the maximum number of saved files -pool_lock=[Lock(),Lock(),Lock(),Lock(),Lock()] #create a list of locks for each process's replay pool -lock=[Lock(),Lock()] #create two locks for synchronization -for p in range(5): #loop over the processes - Process(target=kernel.train,args=(p,lock,pool_lock)).start() #start each process with the train function and pass the process id, the number of episodes, the locks and the pool locks as arguments -``` - -### Stop training and saving when condition is met: -```python -import Note.RL.parallel.kernel as k #import kernel module -import DQN as d #import deep Q-network module -from multiprocessing import Process,Lock,Manager #import multiprocessing tools -dqn=d.DQN(4,128,2) #create neural network object with 4 inputs, 128 hidden units and 2 outputs -kernel=k.kernel(dqn,5) #create kernel object with the network and 5 processes to train -kernel.episode=100 #set the number of episodes to 100 -manager=Manager() #create manager object to share data among processes -kernel.init(manager) #initialize shared data with the manager -kernel.action_count=2 #set the number of actions to 2 -kernel.stop=True #set the flag to stop training when a condition is met -kernel.set_up(epsilon=0.01,pool_size=10000,batch=64,update_step=10,trial_count=10,criterion=200) #set up the hyperparameters for training -kernel.PO=3 #use PO3 algorithm for parallel optimization -pool_lock=[Lock(),Lock(),Lock(),Lock(),Lock()] #create a list of locks for each process's replay pool -lock=[Lock(),Lock(),Lock()] #create three locks for synchronization -for p in range(5): #loop over the processes - Process(target=kernel.train,args=(p,lock,pool_lock)).start() #start each process with the train function and pass the process id, the number of episodes, the locks and the pool locks as arguments -``` - -### Process priority: -```python -import Note.RL.parallel.kernel as k #import kernel module -import DQN as d #import deep Q-network module -from multiprocessing import Process,Lock,Manager #import multiprocessing tools -dqn=d.DQN(4,128,2) #create neural network object with 4 inputs, 128 hidden units and 2 outputs -kernel=k.kernel(dqn,5) #create kernel object with the network and 5 processes to train -kernel.episode=100 #set the number of episodes to 100 -manager=Manager() #create manager object to share data among processes -kernel.priority_flag=True #set the flag to use priority scheduling for processes -kernel.init(manager) #initialize shared data with the manager -kernel.action_count=2 #set the number of actions to 2 -kernel.set_up(epsilon=0.01,pool_size=10000,batch=64,update_step=10) #set up the hyperparameters for training -kernel.PO=3 #use PO3 algorithm for parallel optimization -pool_lock=[Lock(),Lock(),Lock(),Lock(),Lock()] #create a list of locks for each process's replay pool -lock=[Lock(),Lock()] #create two locks for synchronization -for p in range(5): #loop over the processes - Process(target=kernel.train,args=(p,lock,pool_lock)).start() #start each process with the train function and pass the process id, the number of episodes, the locks and the pool locks as arguments -``` - - -# Parallel test: -**You can get a neural network example from the link below, and then you can import neural network and train with kernel, example code are below.** - -https://github.com/NoteDancing/Note-documentation/blob/Note-7.0/Note%207.0%20documentation/DL/neural%20network/tensorflow/parallel/nn.py - -```python -import tensorflow as tf #import tensorflow library -from multiprocessing import Process #import multiprocessing tools -import nn_acc as n #import neural network module with accuracy function -import Note.DL.dl.test as t #import parallel test module -mnist=tf.keras.datasets.mnist #load mnist dataset -(x_train,y_train),(x_test,y_test)=mnist.load_data() #split data into train and test sets -x_train,x_test =x_train/255.0,x_test/255.0 #normalize data -nn=n.nn() #create neural network object -nn.build() #build the network structure -test=t.parallel_test(nn,x_test,y_test,6,32) #create parallel test object with the network, the test data, the number of processes and the batch size -test.segment_data() #segment data for each process -for p in range(6): #loop over the processes - Process(target=test.test).start() #start each process with the test function -loss,acc=test.loss_acc() #calculate the loss and accuracy of the test -``` - - -# Online training: -**You can get a neural network example from the link below, and then you can import neural network and train with kernel, example code are below.** - -https://github.com/NoteDancing/Note-documentation/blob/Note-7.0/Note%207.0%20documentation/DL/neural%20network/tensorflow/non-parallel/nn_ol.py - -**example:** -```python -import Note.DL.kernel as k #import kernel module -import tensorflow as tf #import tensorflow library -import nn_ol as n #import neural network module with online learning -mnist=tf.keras.datasets.mnist #load mnist dataset -(x_train,y_train),(x_test,y_test)=mnist.load_data() #split data into train and test sets -x_train,x_test =x_train/255.0,x_test/255.0 #normalize data -nn=n.nn(x_train,y_train) #create neural network object with train data and labels -kernel=k.kernel(nn) #create kernel object with the network -kernel.platform=tf #set the platform to tensorflow -kernel.data(x_train,y_train) #input train data and labels to the kernel -kernel.train_online() #train the network online -``` - - -# Write neural network class in the interpreter: -```python -import Note.DL.kernel as k #import kernel module -import tensorflow as tf #import tensorflow library - -class nn: - def __init__(self): - self.model=tf.keras.models.Sequential([ - tf.keras.layers.Flatten(input_shape=(28, 28)), - tf.keras.layers.Dense(128,activation='relu'), - tf.keras.layers.Dropout(0.2), - tf.keras.layers.Dense(10) - ]) - self.param=self.model.weights #parameter list,kernel uses it list for backpropagation. - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.opt=tf.keras.optimizers.Adam() #optimizer,kernel uses it to optimize. - - - def fp(self,data): #forward propagation function,kernel uses it for forward propagation. - output=self.model(data) - return output - - - def loss(self,output,labels): #loss functino,kernel uses it to calculate loss. - return self.loss_object(labels,output) - -mnist=tf.keras.datasets.mnist #load mnist dataset -(x_train,y_train),(x_test,y_test)=mnist.load_data() #split data into train and test sets -x_train,x_test =x_train/255.0,x_test/255.0 #normalize data -nn=nn() #create neural network object -kernel=k.kernel(nn) #create kernel object with the network -kernel.platform=tf #set the platform to tensorflow -kernel.data(x_train,y_train) #input train data to the kernel -kernel.train(32,5) #train the network with batch size 32 and epoch 5 -``` - - -# Check neural network: -## DL: -You can test it before using the kernel training neural network. -```python -from Note.DL.dl.check_nn import check #import check function from check_nn module -import tensorflow as tf #import tensorflow library -import nn as n #import neural network module -mnist=tf.keras.datasets.mnist #load mnist dataset -(x_train,y_train),(x_test,y_test)=mnist.load_data() #split data into train and test sets -x_train,x_test =x_train/255.0,x_test/255.0 #normalize data -nn=n.nn() #create neural network object -check(nn,tf,x_train[:32],y_train[:32]) #check the network with tensorflow platform and a sample of 32 data points and labels -``` - -## RL: -You can test it before using the kernel training neural network. -```python -from Note.RL.rl.check_nn import check #import check function from check_nn module -import tensorflow as tf #import tensorflow library -import DQN as d #import deep Q-network module -dqn=d.DQN(4,128,2) #create neural network object with 4 inputs, 128 hidden units and 2 outputs -check(dqn,tf,2) #check the network with tensorflow platform and 2 actions -``` - - -# GPT: -**Layers in the Note.nn.layer.GPT package created by GPT** - -https://github.com/NoteDancing/Note/tree/Note-7.0/Note/nn/layer/GPT From aebb1c92cc37566d581052b20f82702e16c442db Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Wed, 16 Aug 2023 14:12:14 +0800 Subject: [PATCH 175/337] Add files via upload --- .../DL/pytorch/non-parallel/nn.py | 53 +++++++++ 7.0/neuralnetwork/DL/pytorch/parallel/nn.py | 41 +++++++ .../DL/pytorch/parallel/nn_device.py | 45 ++++++++ 7.0/neuralnetwork/DL/tensorflow/layer/LSTM.py | 37 +++++++ .../DL/tensorflow/layer/attn_LSTM.py | 44 ++++++++ 7.0/neuralnetwork/DL/tensorflow/layer/cnn.py | 50 +++++++++ .../DL/tensorflow/layer/cnn_lmix.py | 58 ++++++++++ 7.0/neuralnetwork/DL/tensorflow/layer/nn.py | 27 +++++ .../DL/tensorflow/layer/nn_acc.py | 32 ++++++ .../DL/tensorflow/layer/nn_clipping.py | 40 +++++++ .../DL/tensorflow/layer/self_LSTM.py | 45 ++++++++ .../DL/tensorflow/layer/transformer.py | 46 ++++++++ .../DL/tensorflow/non-parallel/cnn.py | 27 +++++ .../DL/tensorflow/non-parallel/lstm.py | 26 +++++ .../DL/tensorflow/non-parallel/nn.py | 23 ++++ .../DL/tensorflow/non-parallel/nn_acc.py | 28 +++++ .../DL/tensorflow/non-parallel/nn_ol.py | 37 +++++++ .../DL/tensorflow/parallel/LSTM.py | 46 ++++++++ .../DL/tensorflow/parallel/attn_LSTM.py | 53 +++++++++ .../DL/tensorflow/parallel/cnn.py | 59 ++++++++++ .../DL/tensorflow/parallel/cnn_lmix.py | 67 ++++++++++++ .../DL/tensorflow/parallel/nn.py | 39 +++++++ .../DL/tensorflow/parallel/nn_.py | 39 +++++++ .../DL/tensorflow/parallel/nn_acc.py | 44 ++++++++ .../DL/tensorflow/parallel/nn_attenuate.py | 57 ++++++++++ .../DL/tensorflow/parallel/nn_clipping.py | 47 ++++++++ .../DL/tensorflow/parallel/nn_device.py | 40 +++++++ .../DL/tensorflow/parallel/self_LSTM.py | 54 ++++++++++ .../DL/tensorflow/parallel/transformer.py | 55 ++++++++++ .../RL/pytorch/non-parallel/DDPG.py | 101 ++++++++++++++++++ .../RL/pytorch/non-parallel/DQN.py | 64 +++++++++++ .../RL/pytorch/non-parallel/DQN_pr.py | 76 +++++++++++++ .../RL/pytorch/non-parallel/DoubleDQN.py | 65 +++++++++++ .../RL/pytorch/non-parallel/DuelingDQN.py | 67 ++++++++++++ 7.0/neuralnetwork/RL/pytorch/parallel/DQN.py | 65 +++++++++++ .../RL/tensorflow/non-parallel/DDPG.py | 84 +++++++++++++++ .../RL/tensorflow/non-parallel/DQN.py | 46 ++++++++ .../RL/tensorflow/non-parallel/DQN_pr.py | 59 ++++++++++ .../RL/tensorflow/parallel/DQN.py | 52 +++++++++ 39 files changed, 1938 insertions(+) create mode 100644 7.0/neuralnetwork/DL/pytorch/non-parallel/nn.py create mode 100644 7.0/neuralnetwork/DL/pytorch/parallel/nn.py create mode 100644 7.0/neuralnetwork/DL/pytorch/parallel/nn_device.py create mode 100644 7.0/neuralnetwork/DL/tensorflow/layer/LSTM.py create mode 100644 7.0/neuralnetwork/DL/tensorflow/layer/attn_LSTM.py create mode 100644 7.0/neuralnetwork/DL/tensorflow/layer/cnn.py create mode 100644 7.0/neuralnetwork/DL/tensorflow/layer/cnn_lmix.py create mode 100644 7.0/neuralnetwork/DL/tensorflow/layer/nn.py create mode 100644 7.0/neuralnetwork/DL/tensorflow/layer/nn_acc.py create mode 100644 7.0/neuralnetwork/DL/tensorflow/layer/nn_clipping.py create mode 100644 7.0/neuralnetwork/DL/tensorflow/layer/self_LSTM.py create mode 100644 7.0/neuralnetwork/DL/tensorflow/layer/transformer.py create mode 100644 7.0/neuralnetwork/DL/tensorflow/non-parallel/cnn.py create mode 100644 7.0/neuralnetwork/DL/tensorflow/non-parallel/lstm.py create mode 100644 7.0/neuralnetwork/DL/tensorflow/non-parallel/nn.py create mode 100644 7.0/neuralnetwork/DL/tensorflow/non-parallel/nn_acc.py create mode 100644 7.0/neuralnetwork/DL/tensorflow/non-parallel/nn_ol.py create mode 100644 7.0/neuralnetwork/DL/tensorflow/parallel/LSTM.py create mode 100644 7.0/neuralnetwork/DL/tensorflow/parallel/attn_LSTM.py create mode 100644 7.0/neuralnetwork/DL/tensorflow/parallel/cnn.py create mode 100644 7.0/neuralnetwork/DL/tensorflow/parallel/cnn_lmix.py create mode 100644 7.0/neuralnetwork/DL/tensorflow/parallel/nn.py create mode 100644 7.0/neuralnetwork/DL/tensorflow/parallel/nn_.py create mode 100644 7.0/neuralnetwork/DL/tensorflow/parallel/nn_acc.py create mode 100644 7.0/neuralnetwork/DL/tensorflow/parallel/nn_attenuate.py create mode 100644 7.0/neuralnetwork/DL/tensorflow/parallel/nn_clipping.py create mode 100644 7.0/neuralnetwork/DL/tensorflow/parallel/nn_device.py create mode 100644 7.0/neuralnetwork/DL/tensorflow/parallel/self_LSTM.py create mode 100644 7.0/neuralnetwork/DL/tensorflow/parallel/transformer.py create mode 100644 7.0/neuralnetwork/RL/pytorch/non-parallel/DDPG.py create mode 100644 7.0/neuralnetwork/RL/pytorch/non-parallel/DQN.py create mode 100644 7.0/neuralnetwork/RL/pytorch/non-parallel/DQN_pr.py create mode 100644 7.0/neuralnetwork/RL/pytorch/non-parallel/DoubleDQN.py create mode 100644 7.0/neuralnetwork/RL/pytorch/non-parallel/DuelingDQN.py create mode 100644 7.0/neuralnetwork/RL/pytorch/parallel/DQN.py create mode 100644 7.0/neuralnetwork/RL/tensorflow/non-parallel/DDPG.py create mode 100644 7.0/neuralnetwork/RL/tensorflow/non-parallel/DQN.py create mode 100644 7.0/neuralnetwork/RL/tensorflow/non-parallel/DQN_pr.py create mode 100644 7.0/neuralnetwork/RL/tensorflow/parallel/DQN.py diff --git a/7.0/neuralnetwork/DL/pytorch/non-parallel/nn.py b/7.0/neuralnetwork/DL/pytorch/non-parallel/nn.py new file mode 100644 index 00000000..1faec0fc --- /dev/null +++ b/7.0/neuralnetwork/DL/pytorch/non-parallel/nn.py @@ -0,0 +1,53 @@ +import torch +from torch import nn + + +class NeuralNetwork(nn.Module): + def __init__(self): + super().__init__() + self.flatten=nn.Flatten() + self.linear_relu_stack=nn.Sequential( + nn.Linear(28*28, 512), + nn.ReLU(), + nn.Linear(512, 512), + nn.ReLU(), + nn.Linear(512, 10) + ) + + + def forward(self,x): + x=self.flatten(x) + logits=self.linear_relu_stack(x) + return logits + + +class neuralnetwork: + def __init__(self): + if torch.cuda.is_available(): + self.device=torch.device('cuda') + else: + self.device=torch.device('cpu') + self.model=NeuralNetwork().to(self.device) + self.loss_fn=nn.CrossEntropyLoss() + self.opt=torch.optim.SGD(self.model.parameters(),lr=1e-3) #optimizer,kernel uses it to optimize. + + + def fp(self,x): #forward propagation function,kernel uses it for forward propagation. + pred=self.model(x.to(self.device)) + return pred + + + def loss(self,output,labels): #loss functino,kernel uses it to calculate loss. + loss=self.loss_fn(output,labels.to(self.device)) + return loss + + + def backward(self,loss): #backward function,kernel uses it for backpropagation. + self.optim.zero_grad() + loss.backward() + return + + + def opt(self): #opt function,kernel uses it to optimize. + self.optim.step() + return \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/pytorch/parallel/nn.py b/7.0/neuralnetwork/DL/pytorch/parallel/nn.py new file mode 100644 index 00000000..d9b05567 --- /dev/null +++ b/7.0/neuralnetwork/DL/pytorch/parallel/nn.py @@ -0,0 +1,41 @@ +import torch # import the PyTorch library +from torch import nn # import the neural network module from PyTorch + +class NeuralNetwork(nn.Module): # define a class for the neural network model + def __init__(self): # define the constructor method + super().__init__() # call the parent class constructor + self.flatten = nn.Flatten() # define a layer to flatten the input + self.linear_relu_stack=nn.Sequential( # define a stack of linear and relu layers + nn.Linear(28*28,512), # a linear layer with 28*28 input features and 512 output features + nn.ReLU(), # a relu activation function + nn.Linear(512,512), # another linear layer with 512 input and output features + nn.ReLU(), # another relu activation function + nn.Linear(512,10) # a final linear layer with 512 input features and 10 output features + ) + + + def forward(self,x): # define the forward method + x = self.flatten(x) # flatten the input x + logits=self.linear_relu_stack(x) # pass x through the linear relu stack and get the logits + return logits # return the logits + + +class neuralnetwork: # define another class for the neural network object + def __init__(self): # define the constructor method + if torch.cuda.is_available(): # check if cuda is available + self.device=torch.device('cuda') # set the device to cuda + else: + self.device=torch.device('cpu') # otherwise set the device to cpu + self.model=NeuralNetwork().to(self.device) # create an instance of the NeuralNetwork class and move it to the device + self.loss_fn=nn.CrossEntropyLoss() # define the loss function as cross entropy loss + self.opt=[torch.optim.SGD(self.model.parameters(),lr=1e-3) for _ in range(7)] # define a list of three stochastic gradient descent optimizers with learning rate 1e-3 + + + def fp(self,x): # define a method for forward propagation + pred=self.model(x.to(self.device)) # get the predictions from the model by passing x to the device + return pred # return the predictions + + + def loss(self,output,labels): # define a method for calculating the loss + loss=self.loss_fn(output,labels.to(self.device)) # compute the loss by passing output and labels to the device + return loss # return the loss \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/pytorch/parallel/nn_device.py b/7.0/neuralnetwork/DL/pytorch/parallel/nn_device.py new file mode 100644 index 00000000..ad4b8069 --- /dev/null +++ b/7.0/neuralnetwork/DL/pytorch/parallel/nn_device.py @@ -0,0 +1,45 @@ +import torch # import the PyTorch library +from torch import nn # import the neural network module from PyTorch +from Note.nn.parallel.assign_device_pytorch import assign_device # import a custom function to assign device for PyTorch tensors + + +class NeuralNetwork(nn.Module): # define a class for the neural network model + def __init__(self): # define the constructor method + super().__init__() # call the parent class constructor + self.flatten = nn.Flatten() # define a layer to flatten the input tensor + self.linear_relu_stack=nn.Sequential( # define a stack of linear and relu layers + nn.Linear(28*28,512), # define a linear layer with input size 28*28 and output size 512 + nn.ReLU(), # define a relu activation function + nn.Linear(512,512), # define another linear layer with input and output size 512 + nn.ReLU(), # define another relu activation function + nn.Linear(512,10) # define the final linear layer with output size 10 + ) + + + def forward(self,x): # define the forward method for the model + x = self.flatten(x) # flatten the input tensor + logits=self.linear_relu_stack(x) # pass the flattened tensor through the linear relu stack + return logits # return the logits as the output + + +class neuralnetwork: # define another class for the neural network object + def __init__(self): # define the constructor method + if torch.cuda.is_available(): # check if cuda device is available + self.device=torch.device('cuda') # set the device to cuda + else: # otherwise + self.device=torch.device('cpu') # set the device to cpu + self.model=NeuralNetwork().to(self.device) # create an instance of the neural network model and move it to the device + self.loss_fn=nn.CrossEntropyLoss() # define the loss function as cross entropy loss + self.opt=[torch.optim.SGD(self.model.parameters(),lr=1e-3) for _ in range(7)] # define a list of optimizers as stochastic gradient descent with learning rate 1e-3 + + + def fp(self,x,p): # define a method for forward propagation + pred=self.model(x.to(assign_device(p,'GPU'))) # assign the device according to the process index p + # get the predictions from the model by passing the input tensor to the device and then to the model + return pred # return the predictions + + + def loss(self,output,labels,p): # define a method for calculating loss + loss=self.loss_fn(output,labels.to(assign_device(p,'GPU'))) # assign the device according to the process index p + # calculate the loss by passing the output and labels tensors to the device and then to the loss function + return loss # return the loss \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/layer/LSTM.py b/7.0/neuralnetwork/DL/tensorflow/layer/LSTM.py new file mode 100644 index 00000000..272855a3 --- /dev/null +++ b/7.0/neuralnetwork/DL/tensorflow/layer/LSTM.py @@ -0,0 +1,37 @@ +import tensorflow as tf +from Note.nn.layer.LSTM import LSTM +from Note.nn.layer.dense import dense + +# Define a recurrent neural network class with LSTM layers +class lstm: + def __init__(self): + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.opt=tf.keras.optimizers.Adam() + + + def build(self): + # Create two LSTM layers with 50 hidden units each + self.lstm1=LSTM(weight_shape=(3,50),return_sequence=True) # First LSTM layer that returns the full sequence + self.lstm2=LSTM(weight_shape=(50,50),) # Second LSTM layer that returns the last output only + # Create a dense layer with 10 output units + self.dense=dense([50, 10]) + # Store the parameters of the layers in a list + self.param=[self.lstm1.param, + self.lstm2.param, + self.dense.param, + ] + return + + + def fp(self,data): + # Perform forward propagation on the input data + x=self.lstm1.output(data) # First LSTM layer output + x=self.lstm2.output(x) # Second LSTM layer output + output=self.dense.output(x) # Dense layer output + return output + + + def loss(self, output, labels): + # Compute the loss between the output and the labels + return self.loss_object(labels, output) \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/layer/attn_LSTM.py b/7.0/neuralnetwork/DL/tensorflow/layer/attn_LSTM.py new file mode 100644 index 00000000..1d41befd --- /dev/null +++ b/7.0/neuralnetwork/DL/tensorflow/layer/attn_LSTM.py @@ -0,0 +1,44 @@ +import tensorflow as tf +from Note.nn.layer.LSTM import LSTM +from Note.nn.layer.dense import dense +from Note.nn.layer.attention import attention + +# Define a recurrent neural network class with LSTM layers and attention +class attn_lstm: + def __init__(self): + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.opt=tf.keras.optimizers.Adam() + + + def build(self): + # Create two LSTM layers with 50 hidden units each + self.lstm1=LSTM(weight_shape=(3,50),return_sequence=True) # First LSTM layer that returns the full sequence + self.lstm2=LSTM(weight_shape=(50,50),) # Second LSTM layer that returns the last output only + # Create a dense layer with 10 output units and softmax activation + self.dense=dense([100, 10],activation='softmax') + # Create an attention object with weight shape (50, 50) + self.attn=attention((50,50)) + # Store the parameters of the layers and the attention in a list + self.param=[self.lstm1.param, + self.lstm2.param, + self.dense.param, + self.attn.param] # Add the attention parameters to the list + return + + + def fp(self,data): + # Perform forward propagation on the input data + x=self.lstm1.output(data) # First LSTM layer output, shape (B, N, 50) + y=self.lstm2.output(x) # Second LSTM layer output, shape (B, 50) + # Use the attention object to compute the context vector from x and y, shape (B, 50) + context_vector,_,_=self.attn.output(x,y) + # Concatenate the context vector and the second LSTM output, shape (B, 100) + z=tf.concat([context_vector,y],axis=-1) + output=self.dense.output(z) # Dense layer output with softmax activation, shape (B, 10) + return output + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/layer/cnn.py b/7.0/neuralnetwork/DL/tensorflow/layer/cnn.py new file mode 100644 index 00000000..aa2667f8 --- /dev/null +++ b/7.0/neuralnetwork/DL/tensorflow/layer/cnn.py @@ -0,0 +1,50 @@ +import tensorflow as tf +from Note.nn.layer.conv2d import conv2d +from Note.nn.layer.dense import dense +from Note.nn.layer.flatten import flatten + +# Define a convolutional neural network class +class cnn: + def __init__(self): + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.opt=tf.keras.optimizers.Adam() + + + def build(self): + # Create three convolutional layers with relu activations + self.conv1=conv2d(weight_shape=(3,3,3,32),activation='relu') + self.conv2=conv2d(weight_shape=(3,3,32,64),activation='relu') + self.conv3=conv2d(weight_shape=(3,3,64,64),activation='relu') + # Create two dense layers with relu and linear activations + self.dense1=dense([64*4*4,64],activation='relu') + self.dense2=dense([64,10]) + # Store the parameters of the layers in a list + self.param = [self.conv1.weight, + self.conv2.weight, + self.conv3.weight, + self.dense1.weight, + self.dense1.bias, + self.dense2.weight, + self.dense2.bias] + return + + + def fp(self,data): + # Perform forward propagation on the input data + x=tf.nn.conv2d(data,self.conv1.weight,strides=(1,1),padding='SAME') # First convolutional layer + x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # First max pooling layer + x=tf.nn.conv2d(x,self.conv2.weight,strides=(1,1),padding='SAME') # Second convolutional layer + x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # Second max pooling layer + x=tf.nn.conv2d(x,self.conv3.weight,strides=(1,1),padding='SAME') # Third convolutional layer + x=flatten(x) # Flatten the output to a vector + x=tf.nn.relu(tf.matmul(x,self.dense1.weight)+self.dense1.bias) # First dense layer with relu activation + x=tf.nn.dropout(x,rate=0.5) # Apply dropout to prevent overfitting + output=tf.matmul(x,self.dense2.weight)+self.dense2.bias # Output layer with linear activation + output=tf.nn.dropout(output,rate=0.5) # Apply dropout to prevent overfitting + return output + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/layer/cnn_lmix.py b/7.0/neuralnetwork/DL/tensorflow/layer/cnn_lmix.py new file mode 100644 index 00000000..40232919 --- /dev/null +++ b/7.0/neuralnetwork/DL/tensorflow/layer/cnn_lmix.py @@ -0,0 +1,58 @@ +import tensorflow as tf +from Note.nn.layer.conv2d import conv2d +from Note.nn.layer.dense import dense +from Note.nn.layer.flatten import flatten +from Note.nn.layer.LMix import lmix + +# Define a convolutional neural network class with mixup augmentation +class cnn: + def __init__(self): + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.opt=tf.keras.optimizers.Adam() + # Initialize the alpha parameter for mixup + self.alpha=1.0 + + + def build(self): + # Create three convolutional layers with relu activations + self.conv1=conv2d(weight_shape=(3,3,3,32),activation='relu') + self.conv2=conv2d(weight_shape=(3,3,32,64),activation='relu') + self.conv3=conv2d(weight_shape=(3,3,64,64),activation='relu') + # Create two dense layers with relu and linear activations + self.dense1=dense([64*4*4,64],activation='relu') + self.dense2=dense([64,10]) + # Store the parameters of the layers in a list + self.param = [self.conv1.weight, + self.conv2.weight, + self.conv3.weight, + self.dense1.weight, + self.dense1.bias, + self.dense2.weight, + self.dense2.bias] + return + + + def data_func(self,data_batch,labels_batch): + # Apply mixup augmentation on the input data and labels + return lmix(data_batch,labels_batch,self.alpha) + + + def fp(self,data): + # Perform forward propagation on the input data + x=tf.nn.conv2d(data,self.conv1.weight,strides=(1,1),padding='SAME') # First convolutional layer + x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # First max pooling layer + x=tf.nn.conv2d(x,self.conv2.weight,strides=(1,1),padding='SAME') # Second convolutional layer + x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # Second max pooling layer + x=tf.nn.conv2d(x,self.conv3.weight,strides=(1,1),padding='SAME') # Third convolutional layer + x=flatten(x) # Flatten the output to a vector + x=tf.nn.relu(tf.matmul(x,self.dense1.weight)+self.dense1.bias) # First dense layer with relu activation + x=tf.nn.dropout(x,rate=0.5) # Apply dropout to prevent overfitting + output=tf.matmul(x,self.dense2.weight)+self.dense2.bias # Output layer with linear activation + output=tf.nn.dropout(output,rate=0.5) # Apply dropout to prevent overfitting + return output + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/layer/nn.py b/7.0/neuralnetwork/DL/tensorflow/layer/nn.py new file mode 100644 index 00000000..77ed7454 --- /dev/null +++ b/7.0/neuralnetwork/DL/tensorflow/layer/nn.py @@ -0,0 +1,27 @@ +import tensorflow as tf +import Note.nn.layer.dense as d +from Note.nn.layer.flatten import flatten + + +class nn: + def __init__(self): + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.opt=tf.keras.optimizers.Adam() + + + def build(self): + self.layer1=d.dense([784,128],activation='relu') + self.layer2=d.dense([128,10]) + self.param=[self.layer1.param,self.layer2.param] + return + + + def fp(self,data): + data=flatten(data) + output1=self.layer1.output(data) + output2=self.layer2.output(output1) + return output2 + + + def loss(self,output,labels): + return self.loss_object(labels,output) \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/layer/nn_acc.py b/7.0/neuralnetwork/DL/tensorflow/layer/nn_acc.py new file mode 100644 index 00000000..cad20961 --- /dev/null +++ b/7.0/neuralnetwork/DL/tensorflow/layer/nn_acc.py @@ -0,0 +1,32 @@ +import tensorflow as tf +import Note.nn.layer.dense as d +from Note.nn.layer.flatten import flatten + + +class nn: + def __init__(self): + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.train_accuracy=tf.keras.metrics.SparseCategoricalAccuracy(name='train_accuracy') + self.opt=tf.keras.optimizers.Adam() + + + def build(self): + self.layer1=d.dense([784,128],activation='relu') + self.layer2=d.dense([128,10]) + self.param=[self.layer1.param,self.layer2.param] + return + + + def fp(self,data): + data=flatten(data) + output1=self.layer1.output(data) + output2=self.layer2.output(output1) + return output2 + + + def loss(self,output,labels): + return self.loss_object(labels,output) + + + def accuracy(self,output,labels): + return self.train_accuracy(labels,output) \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/layer/nn_clipping.py b/7.0/neuralnetwork/DL/tensorflow/layer/nn_clipping.py new file mode 100644 index 00000000..b53a43da --- /dev/null +++ b/7.0/neuralnetwork/DL/tensorflow/layer/nn_clipping.py @@ -0,0 +1,40 @@ +import tensorflow as tf +import Note.nn.layer.dense as d +from Note.nn.layer.flatten import flatten +from Note.nn.gradient_clipping import gradient_clipping + +# Define a neural network class +class nn: + def __init__(self): + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.opt=tf.keras.optimizers.Adam() + + + def build(self): + # Create two dense layers with relu and linear activations + self.layer1=d.dense([784,128],activation='relu') + self.layer2=d.dense([128,10]) + # Store the parameters of the layers in a list + self.param=[self.layer1.param,self.layer2.param] + return + + + def fp(self,data): + # Perform forward propagation on the input data + data=flatten(data) + output1=self.layer1.output(data) + output2=self.layer2.output(output1) + return output2 + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) + + + def gradient(self,tape,loss): + # Compute the gradients of the loss with respect to the parameters + grads=tape.gradient(loss,self.param) + # Clip the gradients by value to avoid exploding gradients + return gradient_clipping(grads,'value',1.0) \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/layer/self_LSTM.py b/7.0/neuralnetwork/DL/tensorflow/layer/self_LSTM.py new file mode 100644 index 00000000..edaa2420 --- /dev/null +++ b/7.0/neuralnetwork/DL/tensorflow/layer/self_LSTM.py @@ -0,0 +1,45 @@ +import tensorflow as tf +from Note.nn.layer.LSTM import LSTM +from Note.nn.layer.self_attention import self_attention +from Note.nn.layer.dense import dense +from Note.nn.layer.emdedding import embedding + +# Define a LSTM self attention neural network class +class self_LSTM: + def __init__(self,vocab_size,embed_size,num_heads,num_layers): + self.vocab_size=vocab_size + self.embed_size=embed_size + self.num_heads=num_heads + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.opt=tf.keras.optimizers.Adam() + + + def build(self): + # Create an embedding layer to map input tokens to vectors + self.embedding_layer = embedding(self.vocab_size,self.embed_size) + # Create a LSTM layer to process the input sequence + self.lstm_layer = LSTM(weight_shape=[self.embed_size,self.embed_size],return_sequence=True) + # Create a self attention layer to attend to the LSTM output + self.self_attention_layer = self_attention(weight_shape=[self.embed_size,self.embed_size],num_heads=self.num_heads) + # Create a linear layer to map the attention output to logits + self.logits_layer = dense([self.embed_size,self.vocab_size]) + # Store the parameters of the layers in a list + self.param = [self.embedding_layer.embeddings] + self.param.extend(self.lstm_layer.param) + self.param.extend(self.self_attention_layer.param) + self.param.extend(self.logits_layer.param) + + + def fp(self,data): + # Perform forward propagation on the input data + x = self.embedding_layer.output(data) # shape: (batch_size, seq_len, embed_size) + x = self.lstm_layer.output(x) # shape: (batch_size, seq_len, embed_size) + x, attention_weights = self.self_attention_layer.output(x,self.num_heads) # shape: (batch_size, seq_len, embed_size) + output = self.logits_layer.output(x) # shape: (batch_size, seq_len, vocab_size) + return output + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/layer/transformer.py b/7.0/neuralnetwork/DL/tensorflow/layer/transformer.py new file mode 100644 index 00000000..47862006 --- /dev/null +++ b/7.0/neuralnetwork/DL/tensorflow/layer/transformer.py @@ -0,0 +1,46 @@ +import tensorflow as tf +from Note.nn.layer.transformer import transformer +from Note.nn.layer.dense import dense +from Note.nn.layer.emdedding import embedding +from Note.nn.positional_encoding import positional_encoding + +# Define a positional encoding transformer neural network class +class Transformer: + def __init__(self,vocab_size,embed_size,num_heads,num_layers,max_len): + self.vocab_size=vocab_size + self.embed_size=embed_size + self.num_heads=num_heads + self.num_layers=num_layers + self.max_len=max_len + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.opt=tf.keras.optimizers.Adam() + + + def build(self): + # Create an embedding layer to map input tokens to vectors + self.embedding_layer = embedding(self.vocab_size,self.embed_size) + # Create a list of transformer layers + self.transformer_layers = [transformer(weight_shape=[self.embed_size, self.embed_size], num_heads=self.num_heads) for _ in range(self.num_layers)] + # Create a linear layer to map the transformer output to logits + self.logits_layer = dense([self.embed_size,self.vocab_size]) + # Store the parameters of the layers in a list + self.param = [self.embedding_layer.embeddings] + for transformer_layer in self.transformer_layers: + self.param.extend(transformer_layer.param) + self.param.extend(self.logits_layer.param) + + + def fp(self,data): + # Perform forward propagation on the input data + x = self.embedding_layer.output(data) # shape: (batch_size, seq_len, embed_size) + x += positional_encoding(self.max_len,self.embed_size) # shape: (batch_size, seq_len, embed_size) + for transformer_layer in self.transformer_layers: + x = transformer_layer.output(x) # shape: (batch_size, seq_len, embed_size) + output = self.logits_layer.output(x) # shape: (batch_size, seq_len, vocab_size) + return output + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/non-parallel/cnn.py b/7.0/neuralnetwork/DL/tensorflow/non-parallel/cnn.py new file mode 100644 index 00000000..bd7ad1bf --- /dev/null +++ b/7.0/neuralnetwork/DL/tensorflow/non-parallel/cnn.py @@ -0,0 +1,27 @@ +import tensorflow as tf +from tensorflow.keras import layers,models + + +class cnn: + def __init__(self): + self.model=models.Sequential() + self.model.add(layers.Conv2D(32,(3,3),activation='relu',input_shape=(32,32,3))) + self.model.add(layers.MaxPooling2D((2,2))) + self.model.add(layers.Conv2D(64,(3,3),activation='relu')) + self.model.add(layers.MaxPooling2D((2,2))) + self.model.add(layers.Conv2D(64,(3,3),activation='relu')) + self.model.add(layers.Flatten()) + self.model.add(layers.Dense(64,activation='relu')) + self.model.add(layers.Dense(10)) + self.param=self.model.weights + self.opt=tf.keras.optimizers.Adam() + + + def fp(self,data): + output=self.model(data) + return output + + + def loss(self,output,labels): + loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + return loss(labels,output) \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/non-parallel/lstm.py b/7.0/neuralnetwork/DL/tensorflow/non-parallel/lstm.py new file mode 100644 index 00000000..29b9b396 --- /dev/null +++ b/7.0/neuralnetwork/DL/tensorflow/non-parallel/lstm.py @@ -0,0 +1,26 @@ +import tensorflow as tf + + +class lstm: + def __init__(self,encoder): + self.model=tf.keras.Sequential([ + encoder, + tf.keras.layers.Embedding(len(encoder.get_vocabulary()), 64, mask_zero=True), + tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(64, return_sequences=True)), + tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(32)), + tf.keras.layers.Dense(64, activation='relu'), + tf.keras.layers.Dropout(0.5), + tf.keras.layers.Dense(1) + ]) + self.param=self.model.weights[1:] + self.opt=tf.keras.optimizers.Adam() + + + def fp(self,data): + output=self.model(data) + return output + + + def loss(self,output,labels): + bce=tf.keras.losses.BinaryCrossentropy(from_logits=True) + return bce(labels,output) \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/non-parallel/nn.py b/7.0/neuralnetwork/DL/tensorflow/non-parallel/nn.py new file mode 100644 index 00000000..b4f03a92 --- /dev/null +++ b/7.0/neuralnetwork/DL/tensorflow/non-parallel/nn.py @@ -0,0 +1,23 @@ +import tensorflow as tf + + +class nn: + def __init__(self): + self.model=tf.keras.models.Sequential([ + tf.keras.layers.Flatten(input_shape=(28, 28)), + tf.keras.layers.Dense(128,activation='relu'), + tf.keras.layers.Dropout(0.2), + tf.keras.layers.Dense(10) + ]) + self.param=self.model.weights #parameter list,kernel uses it list for backpropagation. + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.opt=tf.keras.optimizers.Adam() #optimizer,kernel uses it to optimize. + + + def fp(self,data): #forward propagation function,kernel uses it for forward propagation. + output=self.model(data) + return output + + + def loss(self,output,labels): #loss functino,kernel uses it to calculate loss. + return self.loss_object(labels,output) \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/non-parallel/nn_acc.py b/7.0/neuralnetwork/DL/tensorflow/non-parallel/nn_acc.py new file mode 100644 index 00000000..0a726603 --- /dev/null +++ b/7.0/neuralnetwork/DL/tensorflow/non-parallel/nn_acc.py @@ -0,0 +1,28 @@ +import tensorflow as tf + +#An example with accuracy function. +class nn: + def __init__(self): + self.model=tf.keras.models.Sequential([ + tf.keras.layers.Flatten(input_shape=(28, 28)), + tf.keras.layers.Dense(128,activation='relu'), + tf.keras.layers.Dropout(0.2), + tf.keras.layers.Dense(10) + ]) + self.param=self.model.weights #parameter list,kernel uses it list for backpropagation. + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.train_accuracy=tf.keras.metrics.SparseCategoricalAccuracy(name='train_accuracy') + self.opt=tf.keras.optimizers.Adam() #optimizer,kernel uses it to optimize. + + + def fp(self,data): #forward propagation function,kernel uses it for forward propagation. + output=self.model(data) + return output + + + def loss(self,output,labels): #loss functino,kernel uses it to calculate loss. + return self.loss_object(labels,output) + + + def accuracy(self,output,labels): #accuracy function,kernel uses it to calculate accuracy. + return self.train_accuracy(labels,output) \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/non-parallel/nn_ol.py b/7.0/neuralnetwork/DL/tensorflow/non-parallel/nn_ol.py new file mode 100644 index 00000000..9dfa608d --- /dev/null +++ b/7.0/neuralnetwork/DL/tensorflow/non-parallel/nn_ol.py @@ -0,0 +1,37 @@ +import tensorflow as tf # import TensorFlow library +import numpy as np # import NumPy library + +# This is an example of online training. +class nn: # define a class for the neural network + def __init__(self,data,labels): # initialize the network with data and labels + self.train_data=data # store the training data + self.train_labels=labels # store the training labels + self.model=tf.keras.models.Sequential([ # create a sequential model with four layers + tf.keras.layers.Flatten(input_shape=(28, 28)), # flatten the input to a vector of 28*28 elements + tf.keras.layers.Dense(128,activation='relu'), # add a dense layer with 128 units and ReLU activation + tf.keras.layers.Dropout(0.2), # add a dropout layer with 0.2 dropout rate to prevent overfitting + tf.keras.layers.Dense(10) # add a dense layer with 10 units for the output logits + ]) + self.param=self.model.weights # parameter list, kernel uses it list for backpropagation + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) # loss object, kernel uses it to calculate loss. Here we use sparse categorical crossentropy with logits as output + self.opt=tf.keras.optimizers.Adam() # optimizer, kernel uses it to optimize. Here we use Adam optimizer + self.train_loss_list=[] # a list to store the training loss values + self.counter=0 # counter to keep track of the number of online updates + self.max_length=1000 # maximum length of train_loss_list + + + def online(self): # This is simulative online function, kernel uses it to get online data and labels + index=np.random.choice(60000,size=[32]) # randomly sample 32 indices from 0 to 59999 + if self.counter==10000: # if the counter reaches 10000, stop the online training + return 'stop' + else: # otherwise, return the data and labels corresponding to the sampled indices + return [self.train_data[index],self.train_labels[index]] + + + def fp(self,data): # forward propagation function, kernel uses it for forward propagation + output=self.model(data) # pass the data through the model and get the output logits + return output + + + def loss(self,output,labels): # loss function, kernel uses it to calculate loss + return self.loss_object(labels,output) # return the sparse categorical crossentropy loss between labels and output \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/LSTM.py b/7.0/neuralnetwork/DL/tensorflow/parallel/LSTM.py new file mode 100644 index 00000000..e8686c10 --- /dev/null +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/LSTM.py @@ -0,0 +1,46 @@ +import tensorflow as tf +from Note.nn.layer.LSTM import LSTM +from Note.nn.layer.dense import dense +from Note.nn.parallel.optimizer import Adam + +# Define a recurrent neural network class with LSTM layers +class lstm: + def __init__(self): + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.optimizer=Adam() + # Initialize a variable to keep track of the batch count + self.bc=tf.Variable(0, dtype=tf.float32) + + + def build(self): + # Create two LSTM layers with 50 hidden units each + self.lstm1=LSTM(weight_shape=(3,50),return_sequence=True) # First LSTM layer that returns the full sequence + self.lstm2=LSTM(weight_shape=(50,50),) # Second LSTM layer that returns the last output only + # Create a dense layer with 10 output units + self.dense=dense([50, 10]) + # Store the parameters of the layers in a list + self.param=[self.lstm1.param, + self.lstm2.param, + self.dense.param, + ] + return + + + def fp(self,data): + # Perform forward propagation on the input data + x=self.lstm1.output(data) # First LSTM layer output + x=self.lstm2.output(x) # Second LSTM layer output + output=self.dense.output(x) # Dense layer output + return output + + + def loss(self, output, labels): + # Compute the loss between the output and the labels + return self.loss_object(labels, output) + + + def opt(self,gradient): + # Perform optimization on the parameters using the gradient and the batch count + param=self.optimizer.opt(gradient,self.param,self.bc[0]) + return param \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/attn_LSTM.py b/7.0/neuralnetwork/DL/tensorflow/parallel/attn_LSTM.py new file mode 100644 index 00000000..198e46b1 --- /dev/null +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/attn_LSTM.py @@ -0,0 +1,53 @@ +import tensorflow as tf +from Note.nn.layer.LSTM import LSTM +from Note.nn.layer.dense import dense +from Note.nn.layer.attention import attention +from Note.nn.parallel.optimizer import Adam + +# Define a recurrent neural network class with LSTM layers and attention +class attn_lstm: + def __init__(self): + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.optimizer=Adam() + # Initialize a variable to keep track of the batch count + self.bc=tf.Variable(0, dtype=tf.float32) + + + def build(self): + # Create two LSTM layers with 50 hidden units each + self.lstm1=LSTM(weight_shape=(3,50),return_sequence=True) # First LSTM layer that returns the full sequence + self.lstm2=LSTM(weight_shape=(50,50),) # Second LSTM layer that returns the last output only + # Create a dense layer with 10 output units and softmax activation + self.dense=dense([100, 10],activation='softmax') + # Create an attention object with weight shape (50, 50) + self.attn=attention((50,50)) + # Store the parameters of the layers and the attention in a list + self.param=[self.lstm1.param, + self.lstm2.param, + self.dense.param, + self.attn.param] # Add the attention parameters to the list + return + + + def fp(self,data): + # Perform forward propagation on the input data + x=self.lstm1.output(data) # First LSTM layer output, shape (B, N, 50) + y=self.lstm2.output(x) # Second LSTM layer output, shape (B, 50) + # Use the attention object to compute the context vector from x and y, shape (B, 50) + context_vector,_,_=self.attn.output(x,y) + # Concatenate the context vector and the second LSTM output, shape (B, 100) + z=tf.concat([context_vector,y],axis=-1) + output=self.dense.output(z) # Dense layer output with softmax activation, shape (B, 10) + return output + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) + + + def opt(self,gradient): + # Perform optimization on the parameters using the gradient and the batch count + param=self.optimizer.opt(gradient,self.param,self.bc[0]) + return param \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/cnn.py b/7.0/neuralnetwork/DL/tensorflow/parallel/cnn.py new file mode 100644 index 00000000..59be2e63 --- /dev/null +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/cnn.py @@ -0,0 +1,59 @@ +import tensorflow as tf +from Note.nn.layer.conv2d import conv2d +from Note.nn.layer.dense import dense +from Note.nn.layer.flatten import flatten +from Note.nn.parallel.optimizer import Adam + +# Define a convolutional neural network class +class cnn: + def __init__(self): + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.optimizer=Adam() + # Initialize a variable to keep track of the batch count + self.bc=tf.Variable(0,dtype=tf.float32) + + + def build(self): + # Create three convolutional layers with relu activations + self.conv1=conv2d(weight_shape=(3,3,3,32),activation='relu') + self.conv2=conv2d(weight_shape=(3,3,32,64),activation='relu') + self.conv3=conv2d(weight_shape=(3,3,64,64),activation='relu') + # Create two dense layers with relu and linear activations + self.dense1=dense([64*4*4,64],activation='relu') + self.dense2=dense([64,10]) + # Store the parameters of the layers in a list + self.param = [self.conv1.weight, + self.conv2.weight, + self.conv3.weight, + self.dense1.weight, + self.dense1.bias, + self.dense2.weight, + self.dense2.bias] + return + + + def fp(self,data): + # Perform forward propagation on the input data + x=tf.nn.conv2d(data,self.conv1.weight,strides=(1,1),padding='SAME') # First convolutional layer + x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # First max pooling layer + x=tf.nn.conv2d(x,self.conv2.weight,strides=(1,1),padding='SAME') # Second convolutional layer + x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # Second max pooling layer + x=tf.nn.conv2d(x,self.conv3.weight,strides=(1,1),padding='SAME') # Third convolutional layer + x=flatten(x) # Flatten the output to a vector + x=tf.nn.relu(tf.matmul(x,self.dense1.weight)+self.dense1.bias) # First dense layer with relu activation + x=tf.nn.dropout(x,rate=0.5) # Apply dropout to prevent overfitting + output=tf.matmul(x,self.dense2.weight)+self.dense2.bias # Output layer with linear activation + output=tf.nn.dropout(output,rate=0.5) # Apply dropout to prevent overfitting + return output + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) + + + def opt(self,gradient): + # Perform optimization on the parameters using the gradient and the batch count + param=self.optimizer.opt(gradient,self.param,self.bc[0]) + return param \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/cnn_lmix.py b/7.0/neuralnetwork/DL/tensorflow/parallel/cnn_lmix.py new file mode 100644 index 00000000..b6e66458 --- /dev/null +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/cnn_lmix.py @@ -0,0 +1,67 @@ +import tensorflow as tf +from Note.nn.layer.conv2d import conv2d +from Note.nn.layer.dense import dense +from Note.nn.layer.flatten import flatten +from Note.nn.parallel.optimizer import Adam +from Note.nn.layer.LMix import lmix + +# Define a convolutional neural network class with mixup augmentation +class cnn: + def __init__(self): + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.optimizer=Adam() + # Initialize the alpha parameter for mixup + self.alpha=1.0 + # Initialize a variable to keep track of the batch count + self.bc=tf.Variable(0,dtype=tf.float32) + + + def build(self): + # Create three convolutional layers with relu activations + self.conv1=conv2d(weight_shape=(3,3,3,32),activation='relu') + self.conv2=conv2d(weight_shape=(3,3,32,64),activation='relu') + self.conv3=conv2d(weight_shape=(3,3,64,64),activation='relu') + # Create two dense layers with relu and linear activations + self.dense1=dense([64*4*4,64],activation='relu') + self.dense2=dense([64,10]) + # Store the parameters of the layers in a list + self.param = [self.conv1.weight, + self.conv2.weight, + self.conv3.weight, + self.dense1.weight, + self.dense1.bias, + self.dense2.weight, + self.dense2.bias] + return + + + def data_func(self,data_batch,labels_batch): + # Apply mixup augmentation on the input data and labels + return lmix(data_batch,labels_batch,self.alpha) + + + def fp(self,data): + # Perform forward propagation on the input data + x=tf.nn.conv2d(data,self.conv1.weight,strides=(1,1),padding='SAME') # First convolutional layer + x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # First max pooling layer + x=tf.nn.conv2d(x,self.conv2.weight,strides=(1,1),padding='SAME') # Second convolutional layer + x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # Second max pooling layer + x=tf.nn.conv2d(x,self.conv3.weight,strides=(1,1),padding='SAME') # Third convolutional layer + x=flatten(x) # Flatten the output to a vector + x=tf.nn.relu(tf.matmul(x,self.dense1.weight)+self.dense1.bias) # First dense layer with relu activation + x=tf.nn.dropout(x,rate=0.5) # Apply dropout to prevent overfitting + output=tf.matmul(x,self.dense2.weight)+self.dense2.bias # Output layer with linear activation + output=tf.nn.dropout(output,rate=0.5) # Apply dropout to prevent overfitting + return output + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) + + + def opt(self,gradient): + # Perform optimization on the parameters using the gradient and the batch count + param=self.optimizer.opt(gradient,self.param,self.bc[0]) + return param \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn.py new file mode 100644 index 00000000..1b16b5c0 --- /dev/null +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/nn.py @@ -0,0 +1,39 @@ +import tensorflow as tf +import Note.nn.layer.dense as d +from Note.nn.parallel.optimizer import Momentum +from Note.nn.layer.flatten import flatten + +# Define a neural network class +class nn: + def __init__(self): + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.optimizer=Momentum(0.07,0.7) + + + def build(self): + # Create two dense layers with relu and linear activations + self.layer1=d.dense([784,128],activation='relu') + self.layer2=d.dense([128,10]) + # Store the parameters of the layers in a list + self.param=[self.layer1.param,self.layer2.param] + return + + + def fp(self,data): + # Perform forward propagation on the input data + data=flatten(data) + output1=self.layer1.output(data) + output2=self.layer2.output(output1) + return output2 + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) + + + def opt(self,gradient): + # Perform optimization on the parameters using the gradient + param=self.optimizer.opt(gradient,self.param) + return param \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_.py new file mode 100644 index 00000000..d115cc0c --- /dev/null +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_.py @@ -0,0 +1,39 @@ +import tensorflow as tf +from Note.nn.parallel.optimizer import Momentum +from Note.nn.layer.flatten import flatten + +# Define a neural network class +class nn: + def __init__(self): + # Initialize the weights and biases of three layers with random values + self.weight1=tf.Variable(tf.random.normal([784,64],dtype='float64')) + self.bias1=tf.Variable(tf.random.normal([64],dtype='float64')) + self.weight2=tf.Variable(tf.random.normal([64,64],dtype='float64')) + self.bias2=tf.Variable(tf.random.normal([64],dtype='float64')) + self.weight3=tf.Variable(tf.random.normal([64,10],dtype='float64')) + self.bias3=tf.Variable(tf.random.normal([10],dtype='float64')) + # Store the parameters of the layers in a list + self.param=[self.weight1,self.weight2,self.weight3,self.bias1,self.bias2,self.bias3] + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.optimizer=Momentum(0.07,0.7) + + + def fp(self,data): + # Perform forward propagation on the input data + data=flatten(data) + layer1=tf.nn.relu(tf.matmul(data,self.weight1)+self.bias1) # First layer with relu activation + layer2=tf.nn.relu(tf.matmul(layer1,self.weight2)+self.bias2) # Second layer with relu activation + output=tf.matmul(layer2,self.weight3)+self.bias3 # Output layer with linear activation + return output + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) + + + def opt(self,gradient): + # Perform optimization on the parameters using the gradient + param=self.optimizer.opt(gradient,self.param) + return param \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_acc.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_acc.py new file mode 100644 index 00000000..19b055dc --- /dev/null +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_acc.py @@ -0,0 +1,44 @@ +import tensorflow as tf +import Note.nn.layer.dense as d +from Note.nn.parallel.optimizer import Momentum +from Note.nn.layer.flatten import flatten +from Note.nn.accuracy import sparse_categorical_accuracy + +# Define a neural network class +class nn: + def __init__(self): + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.optimizer=Momentum(0.07,0.7) + + + def build(self): + # Create two dense layers with relu and linear activations + self.layer1=d.dense([784,128],activation='relu') + self.layer2=d.dense([128,10]) + # Store the parameters of the layers in a list + self.param=[self.layer1.param,self.layer2.param] + return + + + def fp(self,data): + # Perform forward propagation on the input data + data=flatten(data) + output1=self.layer1.output(data) + output2=self.layer2.output(output1) + return output2 + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) + + + def accuracy(self,output,labels): #accuracy function,kernel uses it to calculate accuracy. + return sparse_categorical_accuracy(labels,output) + + + def opt(self,gradient): + # Perform optimization on the parameters using the gradient + param=self.optimizer.opt(gradient,self.param) + return param \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_attenuate.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_attenuate.py new file mode 100644 index 00000000..2f176151 --- /dev/null +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_attenuate.py @@ -0,0 +1,57 @@ +import tensorflow as tf +from tensorflow.python.util import nest +import Note.nn.layer.dense as d +from Note.nn.parallel.optimizer import Momentum +from Note.nn.layer.flatten import flatten +from Note.nn.accuracy import sparse_categorical_accuracy + +# Define a neural network class with gradient attenuation +class nn: + def __init__(self): + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.optimizer=Momentum(0.07,0.7) + # Initialize a variable to keep track of the number of optimization steps + self.opt_counter=tf.Variable(tf.zeros(3,dtype=tf.float32)) + + + def build(self): + # Create two dense layers with relu and linear activations + self.layer1=d.dense([784,128],activation='relu') + self.layer2=d.dense([128,10]) + # Store the parameters of the layers in a list + self.param=[self.layer1.param,self.layer2.param] + return + + + def fp(self,data): + # Perform forward propagation on the input data + data=flatten(data) # Flatten the data to a one-dimensional vector + output1=self.layer1.output(data) # Apply the first layer to the data and get the output + output2=self.layer2.output(output1) # Apply the second layer to the output of the first layer and get the final output + return output2 + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) # Use the loss function to calculate the loss + + + def accuracy(self,output,labels): #accuracy function,kernel uses it to calculate accuracy. + return sparse_categorical_accuracy(labels,output) + + + def attenuate(self,gradient,oc,p): #gradient attenuation function,kernel uses it to calculate attenuation coefficient. + # Apply an exponential decay to the gradient based on the optimization counter + ac=0.9**oc[0][p] #ac:attenuation coefficient + gradient_flat=nest.flatten(gradient) # Flatten the gradient to a one-dimensional vector + for i in range(len(gradient_flat)): #oc:optimization counter + gradient_flat[i]=ac*gradient_flat[i] #p:process number + gradient=nest.pack_sequence_as(gradient,gradient_flat) # Restore the gradient to its original shape + return gradient + + + def opt(self,gradient): + # Perform optimization on the parameters using the gradient + param=self.optimizer.opt(gradient,self.param) # Use the optimizer to update the parameters + return param \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_clipping.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_clipping.py new file mode 100644 index 00000000..a0f757d3 --- /dev/null +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_clipping.py @@ -0,0 +1,47 @@ +import tensorflow as tf +import Note.nn.layer.dense as d +from Note.nn.parallel.optimizer import Momentum +from Note.nn.layer.flatten import flatten +from Note.nn.gradient_clipping import gradient_clipping + +# Define a neural network class +class nn: + def __init__(self): + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.optimizer=Momentum(0.07,0.7) + + + def build(self): + # Create two dense layers with relu and linear activations + self.layer1=d.dense([784,128],activation='relu') + self.layer2=d.dense([128,10]) + # Store the parameters of the layers in a list + self.param=[self.layer1.param,self.layer2.param] + return + + + def fp(self,data): + # Perform forward propagation on the input data + data=flatten(data) + output1=self.layer1.output(data) + output2=self.layer2.output(output1) + return output2 + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) + + + def gradient(self,tape,loss): + # Compute the gradients of the loss with respect to the parameters + grads=tape.gradient(loss,self.param) + # Clip the gradients by value to avoid exploding gradients + return gradient_clipping(grads,'value',1.0) + + + def opt(self,gradient): + # Perform optimization on the parameters using the gradient + param=self.optimizer.opt(gradient,self.param) + return param \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_device.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_device.py new file mode 100644 index 00000000..e48acac3 --- /dev/null +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_device.py @@ -0,0 +1,40 @@ +import tensorflow as tf # import TensorFlow library +import Note.nn.layer.dense as d # import Note's dense layer module +from Note.nn.layer.flatten import flatten # import Note's flatten layer function +from Note.nn.parallel.optimizer import Momentum # import Note's momentum optimizer module +from Note.nn.parallel.assign_device import assign_device # import the function to assign device according to the process index and the device type + + +class nn: # A neural network class example, allocate device for multiple threads + def __init__(self): # initialize the network + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) # loss object, kernel uses it to calculate loss. Here we use sparse categorical crossentropy with logits as output + self.optimizer=Momentum(0.07,0.7) # optimizer, kernel uses it to optimize. Here we use a custom momentum optimizer with learning rate 0.07 and momentum 0.7 + + + def build(self): # build function, kernel uses it to create the network layers + # Create two dense layers with relu and linear activations + self.layer1=d.dense([784,128],activation='relu') # the first layer with 784 input units and 128 output units and ReLU activation + self.layer2=d.dense([128,10]) # the second layer with 128 input units and 10 output units and linear activation + # Store the parameters of the layers in a list + self.param=[self.layer1.param,self.layer2.param] # parameter list of both layers, kernel uses it list for backpropagation + return + + + def fp(self,data,p): # forward propagation function, kernel uses it for forward propagation + with tf.device(assign_device(p,'GPU')): # assign the device according to the process index p + data=flatten(data) # flatten the data to a vector of 784 elements + output1=self.layer1.output(data) # pass the data through the first layer and get the output + output2=self.layer2.output(output1) # pass the output of the first layer through the second layer and get the final output logits + return output2 + + + def loss(self,output,labels,p): # loss function, kernel uses it to calculate loss + with tf.device(assign_device(p,'GPU')): # assign the device according to the process index p + return self.loss_object(labels,output) # return the mean softmax cross entropy loss between labels and output + + + def opt(self,gradient,p): # optimization function, kernel uses it to optimize parameter + # Perform optimization on the parameters using the gradient + with tf.device(assign_device(p,'GPU')): # assign the device according to the process index p + param=self.optimizer.opt(gradient,self.param) # apply the Note's momentum optimizer to update the parameters using the gradient + return param \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/self_LSTM.py b/7.0/neuralnetwork/DL/tensorflow/parallel/self_LSTM.py new file mode 100644 index 00000000..85af312a --- /dev/null +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/self_LSTM.py @@ -0,0 +1,54 @@ +import tensorflow as tf +from Note.nn.parallel.optimizer import Adam +from Note.nn.layer.LSTM import LSTM +from Note.nn.layer.self_attention import self_attention +from Note.nn.layer.dense import dense +from Note.nn.layer.emdedding import embedding + +# Define a LSTM self attention neural network class +class self_LSTM: + def __init__(self,vocab_size,embed_size,num_heads,num_layers): + self.vocab_size=vocab_size + self.embed_size=embed_size + self.num_heads=num_heads + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.optimizer=Adam() + # Initialize a variable to keep track of the batch count + self.bc=tf.Variable(0,dtype=tf.float32) + + + def build(self): + # Create an embedding layer to map input tokens to vectors + self.embedding_layer = embedding(self.vocab_size,self.embed_size) + # Create a LSTM layer to process the input sequence + self.lstm_layer = LSTM(weight_shape=[self.embed_size,self.embed_size],return_sequence=True) + # Create a self attention layer to attend to the LSTM output + self.self_attention_layer = self_attention(weight_shape=[self.embed_size,self.embed_size],num_heads=self.num_heads) + # Create a linear layer to map the attention output to logits + self.logits_layer = dense([self.embed_size,self.vocab_size]) + # Store the parameters of the layers in a list + self.param = [self.embedding_layer.embeddings] + self.param.extend(self.lstm_layer.param) + self.param.extend(self.self_attention_layer.param) + self.param.extend(self.logits_layer.param) + + + def fp(self,data): + # Perform forward propagation on the input data + x = self.embedding_layer.output(data) # shape: (batch_size, seq_len, embed_size) + x = self.lstm_layer.output(x) # shape: (batch_size, seq_len, embed_size) + x, attention_weights = self.self_attention_layer.output(x,self.num_heads) # shape: (batch_size, seq_len, embed_size) + output = self.logits_layer.output(x) # shape: (batch_size, seq_len, vocab_size) + return output + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) + + + def opt(self,gradient): + # Perform optimization on the parameters using the gradient and the batch count + param=self.optimizer.opt(gradient,self.param,self.bc[0]) + return param \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/transformer.py b/7.0/neuralnetwork/DL/tensorflow/parallel/transformer.py new file mode 100644 index 00000000..ae052538 --- /dev/null +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/transformer.py @@ -0,0 +1,55 @@ +import tensorflow as tf +from Note.nn.parallel.optimizer import Adam +from Note.nn.layer.transformer import transformer +from Note.nn.layer.dense import dense +from Note.nn.layer.emdedding import embedding +from Note.nn.positional_encoding import positional_encoding + +# Define a positional encoding transformer neural network class +class Transformer: + def __init__(self,vocab_size,embed_size,num_heads,num_layers,max_len): + self.vocab_size=vocab_size + self.embed_size=embed_size + self.num_heads=num_heads + self.num_layers=num_layers + self.max_len=max_len + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.optimizer=Adam() + # Initialize a variable to keep track of the batch count + self.bc=tf.Variable(0,dtype=tf.float32) + + + def build(self): + # Create an embedding layer to map input tokens to vectors + self.embedding_layer = embedding(self.vocab_size,self.embed_size) + # Create a list of transformer layers + self.transformer_layers = [transformer(weight_shape=[self.embed_size, self.embed_size], num_heads=self.num_heads) for _ in range(self.num_layers)] + # Create a linear layer to map the transformer output to logits + self.logits_layer = dense([self.embed_size,self.vocab_size]) + # Store the parameters of the layers in a list + self.param = [self.embedding_layer.embeddings] + for transformer_layer in self.transformer_layers: + self.param.extend(transformer_layer.param) + self.param.extend(self.logits_layer.param) + + + def fp(self,data): + # Perform forward propagation on the input data + x = self.embedding_layer.output(data) # shape: (batch_size, seq_len, embed_size) + x += positional_encoding(self.max_len,self.embed_size) # shape: (batch_size, seq_len, embed_size) + for transformer_layer in self.transformer_layers: + x = transformer_layer.output(x) # shape: (batch_size, seq_len, embed_size) + output = self.logits_layer.output(x) # shape: (batch_size, seq_len, vocab_size) + return output + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) + + + def opt(self,gradient): + # Perform optimization on the parameters using the gradient and the batch count + param=self.optimizer.opt(gradient,self.param,self.bc[0]) + return param \ No newline at end of file diff --git a/7.0/neuralnetwork/RL/pytorch/non-parallel/DDPG.py b/7.0/neuralnetwork/RL/pytorch/non-parallel/DDPG.py new file mode 100644 index 00000000..4b0abd94 --- /dev/null +++ b/7.0/neuralnetwork/RL/pytorch/non-parallel/DDPG.py @@ -0,0 +1,101 @@ +import torch +import torch.nn.functional as F +import numpy as np +import gym + + +class actor(torch.nn.Module): + def __init__(self,state_dim,hidden_dim,action_dim,action_bound): + super(actor,self).__init__() + self.fc1=torch.nn.Linear(state_dim,hidden_dim) + self.fc2=torch.nn.Linear(hidden_dim,action_dim) + self.action_bound=action_bound + + + def forward(self,x): + x=F.relu(self.fc1(x)) + return torch.tanh(self.fc2(x))*self.action_bound + + +class critic(torch.nn.Module): + def __init__(self,state_dim,hidden_dim,action_dim): + super(critic,self).__init__() + self.fc1=torch.nn.Linear(state_dim+action_dim,hidden_dim) + self.fc2=torch.nn.Linear(hidden_dim,1) + + + def forward(self,x,a): + cat=torch.cat([x,a],dim=1) + x=F.relu(self.fc1(cat)) + return self.fc2(x) + + +class DDPG: + def __init__(self,hidden_dim,sigma,gamma,tau,actor_lr,critic_lr): + if torch.cuda.is_available(): + self.device=torch.device('cuda') + else: + self.device=torch.device('cpu') + self.genv=gym.make('Pendulum-v0') + state_dim=self.genv.observation_space.shape[0] + action_dim=self.genv.action_space.shape[0] + action_bound=self.genv.action_space.high[0] + self.actor=actor(state_dim,hidden_dim,action_dim,action_bound).to(self.device) + self.critic=critic(state_dim,hidden_dim,action_dim).to(self.device) + self.target_actor=actor(state_dim,hidden_dim,action_dim,action_bound).to(self.device) + self.target_critic=critic(state_dim,hidden_dim,action_dim).to(self.device) + self.target_actor.load_state_dict(self.actor.state_dict()) + self.target_critic.load_state_dict(self.critic.state_dict()) + self.sigma=sigma + self.gamma=gamma + self.tau=tau + self.actor_opt=torch.optim.Adam(self.actor.parameters(),lr=actor_lr) + self.critic_opt=torch.optim.Adam(self.critic.parameters(),lr=critic_lr) + + + def noise(self): + return np.random.normal(scale=self.sigma) + + + def env(self,a=None,initial=None): + if initial==True: + state=self.genv.reset(seed=0) + return state + else: + next_state,reward,done,_=self.genv.step(a) + return next_state,reward,done + + + def loss(self,s,a,next_s,r,d): + s=torch.tensor(s,dtype=torch.float).to(self.device) + a=torch.tensor(a,dtype=torch.float).view(-1,1).to(self.device) + next_s=torch.tensor(next_s,dtype=torch.float).to(self.device) + r=torch.tensor(r,dtype=torch.float).view(-1,1).to(self.device) + d=torch.tensor(d,dtype=torch.float).view(-1,1).to(self.device) + next_q_value=self.target_critic(next_s,self.target_actor(next_s)) + q_target=r+self.gamma*next_q_value*(1-d) + actor_loss=-torch.mean(self.critic(s,self.actor(s))) + critic_loss=F.mse_loss(self.critic(s,a),q_target) + return [actor_loss,critic_loss] + + + def backward(self,loss): + self.actor_opt.zero_grad() + loss[0].backward() + self.critic_opt.zero_grad() + loss[1].backward() + return + + + def opt(self): + self.actor_opt.step() + self.critic_opt.step() + return + + + def update_param(self): + for target_param,param in zip(self.target_actor.parameters(),self.actor.parameters()): + target_param.data.copy_(target_param.data*(1.0-self.tau)+param.data*self.tau) + for target_param,param in zip(self.target_critic.parameters(),self.critic.parameters()): + target_param.data.copy_(target_param.data*(1.0-self.tau)+param.data*self.tau) + return \ No newline at end of file diff --git a/7.0/neuralnetwork/RL/pytorch/non-parallel/DQN.py b/7.0/neuralnetwork/RL/pytorch/non-parallel/DQN.py new file mode 100644 index 00000000..a0e58689 --- /dev/null +++ b/7.0/neuralnetwork/RL/pytorch/non-parallel/DQN.py @@ -0,0 +1,64 @@ +import torch +import gym +import torch.nn.functional as F + + +class Qnet(torch.nn.Module): + def __init__(self,state_dim,hidden_dim,action_dim): + super(Qnet,self).__init__() + self.fc1=torch.nn.Linear(state_dim,hidden_dim) + self.fc2=torch.nn.Linear(hidden_dim,action_dim) + + + def forward(self,x): + x=F.relu(self.fc1(x)) + return self.fc2(x) + + +class DQN: + def __init__(self,state_dim,hidden_dim,action_dim): + if torch.cuda.is_available(): + self.device=torch.device('cuda') + else: + self.device=torch.device('cpu') + self.nn=Qnet(state_dim,hidden_dim,action_dim).to(self.device) + self.target_q_net=Qnet(state_dim,hidden_dim,action_dim).to(self.device) + self.optimizer=torch.optim.Adam(self.nn.parameters(),lr=2e-3) #optimizer,kernel uses it to optimize. + self.genv=gym.make('CartPole-v0') #create environment + + + def env(self,a=None,initial=None): #environment function,kernel uses it to interact with the environment. + if initial==True: + state=self.genv.reset() + return state + else: + next_state,reward,done,_=self.genv.step(a) + return next_state,reward,done + + + def loss(self,s,a,next_s,r,d): #loss function,kernel uses it to calculate loss. + s=torch.tensor(s,dtype=torch.float).to(self.device) + a=torch.tensor(a,dtype=torch.int64).view(-1,1).to(self.device) + next_s=torch.tensor(next_s,dtype=torch.float).to(self.device) + r=torch.tensor(r,dtype=torch.float).view(-1,1).to(self.device) + d=torch.tensor(d,dtype=torch.float).view(-1,1).to(self.device) + q_value=self.nn(s).gather(1,a) + next_q_value=self.target_q_net(next_s).max(1)[0].view(-1,1) + target=r+0.98*next_q_value*(1-d) + return F.mse_loss(q_value,target) + + + def backward(self,loss): #backward function,kernel uses it for backpropagation. + self.optimizer.zero_grad() + loss.backward() + return + + + def opt(self): #opt function,kernel uses it to optimize. + self.optimizer.step() + return + + + def update_param(self): #update function,kernel uses it to update parameter. + self.target_q_net.load_state_dict(self.nn.state_dict()) + return \ No newline at end of file diff --git a/7.0/neuralnetwork/RL/pytorch/non-parallel/DQN_pr.py b/7.0/neuralnetwork/RL/pytorch/non-parallel/DQN_pr.py new file mode 100644 index 00000000..66c71847 --- /dev/null +++ b/7.0/neuralnetwork/RL/pytorch/non-parallel/DQN_pr.py @@ -0,0 +1,76 @@ +import torch +import gym +import torch.nn.functional as F +import Note.create.RL.rl.prioritized_replay as pr + +#prioritized replay example +class Qnet(torch.nn.Module): + def __init__(self,state_dim,hidden_dim,action_dim): + super(Qnet,self).__init__() + self.fc1=torch.nn.Linear(state_dim,hidden_dim) + self.fc2=torch.nn.Linear(hidden_dim,action_dim) + + + def forward(self,x): + x=F.relu(self.fc1(x)) + return self.fc2(x) + + +class DQN: + def __init__(self,state_dim,hidden_dim,action_dim): + if torch.cuda.is_available(): + self.device=torch.device('cuda') + else: + self.device=torch.device('cpu') + self.nn=Qnet(state_dim,hidden_dim,action_dim).to(self.device) + self.target_q_net=Qnet(state_dim,hidden_dim,action_dim).to(self.device) + self.pr=pr.pr() + self.initial_TD=7 + self._epsilon=0.0007 + self.alpha=0.7 + self.optimizer=torch.optim.Adam(self.nn.parameters(),lr=2e-3) + self.genv=gym.make('CartPole-v0') + + + def env(self,a=None,initial=None): + if initial==True: + state=self.genv.reset(seed=0) + return state + else: + next_state,reward,done,_=self.genv.step(a) + return next_state,reward,done + + + def data_func(self,state_pool,action_pool,next_state_pool,reward_pool,done_pool,batch): + s,a,next_s,r,d=self.pr.sample(state_pool,action_pool,next_state_pool,reward_pool,done_pool,self.epsilon,self.alpha,batch) + return s,a,next_s,r,d + + + def loss(self,s,a,next_s,r,d): + s=torch.tensor(s,dtype=torch.float).to(self.device) + a=torch.tensor(a,dtype=torch.int64).view(-1,1).to(self.device) + next_s=torch.tensor(next_s,dtype=torch.float).to(self.device) + r=torch.tensor(r,dtype=torch.float).view(-1,1).to(self.device) + d=torch.tensor(d,dtype=torch.float).view(-1,1).to(self.device) + q_value=self.nn(s).gather(1,a) + next_q_value=self.target_q_net(next_s).max(1)[0].view(-1,1) + target=r+0.98*next_q_value*(1-d) + TD=target-q_value + self.pr.update_TD(TD) + return torch.mean(TD**2) + + + def backward(self,loss): + self.optimizer.zero_grad() + loss.backward() + return + + + def opt(self): + self.optimizer.step() + return + + + def update_param(self): + self.target_q_net.load_state_dict(self.nn.state_dict()) + return \ No newline at end of file diff --git a/7.0/neuralnetwork/RL/pytorch/non-parallel/DoubleDQN.py b/7.0/neuralnetwork/RL/pytorch/non-parallel/DoubleDQN.py new file mode 100644 index 00000000..31e606e8 --- /dev/null +++ b/7.0/neuralnetwork/RL/pytorch/non-parallel/DoubleDQN.py @@ -0,0 +1,65 @@ +import torch +import gym +import torch.nn.functional as F + + +class Qnet(torch.nn.Module): + def __init__(self,state_dim,hidden_dim,action_dim): + super(Qnet,self).__init__() + self.fc1=torch.nn.Linear(state_dim,hidden_dim) + self.fc2=torch.nn.Linear(hidden_dim,action_dim) + + + def forward(self,x): + x=F.relu(self.fc1(x)) + return self.fc2(x) + + +class DoubleDQN: + def __init__(self,state_dim,hidden_dim,action_dim): + if torch.cuda.is_available(): + self.device=torch.device('cuda') + else: + self.device=torch.device('cpu') + self.nn=Qnet(state_dim,hidden_dim,action_dim).to(self.device) + self.target_q_net=Qnet(state_dim,hidden_dim,action_dim).to(self.device) + self.optimizer=torch.optim.Adam(self.nn.parameters(),lr=2e-3) + self.genv=gym.make('CartPole-v0') + + + def env(self,a=None,initial=None): + if initial==True: + state=self.genv.reset(seed=0) + return state + else: + next_state,reward,done,_=self.genv.step(a) + return next_state,reward,done + + + def loss(self,s,a,next_s,r,d): + s=torch.tensor(s,dtype=torch.float).to(self.device) + a=torch.tensor(a,dtype=torch.int64).view(-1,1).to(self.device) + next_s=torch.tensor(next_s,dtype=torch.float).to(self.device) + r=torch.tensor(r,dtype=torch.float).view(-1,1).to(self.device) + d=torch.tensor(d,dtype=torch.float).view(-1,1).to(self.device) + q_value=self.nn(s).gather(1,a) + max_action=self.nn(next_s).max(1)[1].view(-1,1) + next_q_value=self.target_q_net(next_s).gather(1,max_action) + target=r+0.98*next_q_value*(1-d) + return F.mse_loss(q_value,target) + + + def backward(self,loss): + self.optimizer.zero_grad() + loss.backward() + return + + + def opt(self): + self.optimizer.step() + return + + + def update_param(self): + self.target_q_net.load_state_dict(self.nn.state_dict()) + return \ No newline at end of file diff --git a/7.0/neuralnetwork/RL/pytorch/non-parallel/DuelingDQN.py b/7.0/neuralnetwork/RL/pytorch/non-parallel/DuelingDQN.py new file mode 100644 index 00000000..b81eeb37 --- /dev/null +++ b/7.0/neuralnetwork/RL/pytorch/non-parallel/DuelingDQN.py @@ -0,0 +1,67 @@ +import torch +import gym +import torch.nn.functional as F + + +class VAnet(torch.nn.Module): + def __init__(self,state_dim,hidden_dim,action_dim): + super(VAnet,self).__init__() + self.fc1=torch.nn.Linear(state_dim,hidden_dim) + self.fc_A=torch.nn.Linear(hidden_dim,action_dim) + self.fc_V=torch.nn.Linear(hidden_dim,1) + + + def forward(self,x): + A=self.fc_A(F.relu(self.fc1(x))) + V=self.fc_V(F.relu(self.fc1(x))) + Q=V+A-A.mean(1).view(-1,1) + return Q + + +class DuelingDQN: + def __init__(self,state_dim,hidden_dim,action_dim): + if torch.cuda.is_available(): + self.device=torch.device('cuda') + else: + self.device=torch.device('cpu') + self.nn=VAnet(state_dim,hidden_dim,action_dim).to(self.device) + self.target_q_net=VAnet(state_dim,hidden_dim,action_dim).to(self.device) + self.optimizer=torch.optim.Adam(self.nn.parameters(),lr=2e-3) + self.genv=gym.make('CartPole-v0') + + + def env(self,a=None,initial=None): + if initial==True: + state=self.genv.reset(seed=0) + return state + else: + next_state,reward,done,_=self.genv.step(a) + return next_state,reward,done + + + def loss(self,s,a,next_s,r,d): + s=torch.tensor(s,dtype=torch.float).to(self.device) + a=torch.tensor(a,dtype=torch.int64).view(-1,1).to(self.device) + next_s=torch.tensor(next_s,dtype=torch.float).to(self.device) + r=torch.tensor(r,dtype=torch.float).view(-1,1).to(self.device) + d=torch.tensor(d,dtype=torch.float).view(-1,1).to(self.device) + q_value=self.nn(s).gather(1,a) + next_q_value=self.target_q_net(next_s).max(1)[0].view(-1,1) + target=r+0.98*next_q_value*(1-d) + return F.mse_loss(q_value,target) + + + def backward(self,loss): + self.optimizer.zero_grad() + loss.backward() + return + + + def opt(self): + self.optimizer.step() + return + + + def update_param(self): + self.target_q_net.load_state_dict(self.nn.state_dict()) + return \ No newline at end of file diff --git a/7.0/neuralnetwork/RL/pytorch/parallel/DQN.py b/7.0/neuralnetwork/RL/pytorch/parallel/DQN.py new file mode 100644 index 00000000..f4d8417d --- /dev/null +++ b/7.0/neuralnetwork/RL/pytorch/parallel/DQN.py @@ -0,0 +1,65 @@ +import torch +import gym +import torch.nn.functional as F +from Note.nn.parallel.assign_device_pytorch import assign_device + + +class Qnet(torch.nn.Module): + def __init__(self,state_dim,hidden_dim,action_dim): + super(Qnet,self).__init__() + self.fc1=torch.nn.Linear(state_dim,hidden_dim) + self.fc2=torch.nn.Linear(hidden_dim,action_dim) + + + def forward(self,x): + x=F.relu(self.fc1(x)) + return self.fc2(x) + + +class DQN: + def __init__(self,state_dim,hidden_dim,action_dim): + if torch.cuda.is_available(): + self.device=torch.device('cuda') + else: + self.device=torch.device('cpu') + self.nn=Qnet(state_dim,hidden_dim,action_dim).to(self.device) + self.target_q_net=Qnet(state_dim,hidden_dim,action_dim).to(self.device) + self.optimizer=[torch.optim.Adam(self.nn.parameters(),lr=2e-3) for _ in range(5)] #optimizer,kernel uses it to optimize. + self.genv=[gym.make('CartPole-v0') for _ in range(5)] #create environment + + + def env(self,a=None,p=None,initial=None): #environment function,kernel uses it to interact with the environment. + if initial==True: + state=self.genv[p].reset() + return state + else: + next_state,reward,done,_=self.genv[p].step(a) + return next_state,reward,done + + + def loss(self,s,a,next_s,r,d,p): #loss function,kernel uses it to calculate loss. + s=torch.tensor(s,dtype=torch.float).to(assign_device(p,'GPU')) + a=torch.tensor(a,dtype=torch.int64).view(-1,1).to(assign_device(p,'GPU')) + next_s=torch.tensor(next_s,dtype=torch.float).to(assign_device(p,'GPU')) + r=torch.tensor(r,dtype=torch.float).view(-1,1).to(assign_device(p,'GPU')) + d=torch.tensor(d,dtype=torch.float).view(-1,1).to(assign_device(p,'GPU')) + q_value=self.nn(s).gather(1,a) + next_q_value=self.target_q_net(next_s).max(1)[0].view(-1,1) + target=r+0.98*next_q_value*(1-d) + return F.mse_loss(q_value,target) + + + def backward(self,loss,p): #backward function,kernel uses it for backpropagation. + self.optimizer[p].zero_grad(set_to_none=True) + loss.backward() + return + + + def opt(self,p): #opt function,kernel uses it to optimize. + self.optimizer[p].step() + return + + + def update_param(self): #update function,kernel uses it to update parameter. + self.target_q_net.load_state_dict(self.nn.state_dict()) + return \ No newline at end of file diff --git a/7.0/neuralnetwork/RL/tensorflow/non-parallel/DDPG.py b/7.0/neuralnetwork/RL/tensorflow/non-parallel/DDPG.py new file mode 100644 index 00000000..d3666ee1 --- /dev/null +++ b/7.0/neuralnetwork/RL/tensorflow/non-parallel/DDPG.py @@ -0,0 +1,84 @@ +import tensorflow as tf # import TensorFlow library +import numpy as np # import NumPy library +import gym # import OpenAI Gym library + + +class actor: # define a class for the actor network + def __init__(self,state_dim,hidden_dim,action_dim,action_bound): # initialize the network with state dimension, hidden dimension, action dimension and action bound + self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) # create a weight matrix for the first layer + self.bias1=tf.Variable(tf.random.normal([hidden_dim])) # create a bias vector for the first layer + self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) # create a weight matrix for the second layer + self.bias2=tf.Variable(tf.random.normal([action_dim])) # create a bias vector for the second layer + self.action_bound=action_bound # store the action bound + self.param=[self.weight1,self.bias1,self.weight2,self.bias2] # store the network parameters in a list + + + def fp(self,x): # forward propagation function, kernel uses it for forward propagation + x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) # apply the first layer with ReLU activation + return tf.nn.tanh(tf.matmul(x,self.weight2)+self.bias2)*self.action_bound # apply the second layer with tanh activation and scale it by the action bound + + +class critic: # define a class for the critic network + def __init__(self,state_dim,hidden_dim,action_dim): # initialize the network with state dimension, hidden dimension and action dimension + self.weight1=tf.Variable(tf.random.normal([state_dim+action_dim,hidden_dim])) # create a weight matrix for the first layer + self.bias1=tf.Variable(tf.random.normal([hidden_dim])) # create a bias vector for the first layer + self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) # create a weight matrix for the second layer + self.bias2=tf.Variable(tf.random.normal([action_dim])) # create a bias vector for the second layer + self.param=[self.weight1,self.bias1,self.weight2,self.bias2] # store the network parameters in a list + + + def fp(self,x,a): # forward propagation function, kernel uses it for forward propagation + cat=tf.concat([x,a],axis=1) # concatenate the state and action vectors along the first axis + x=tf.nn.relu(tf.matmul(cat,self.weight1)+self.bias1) # apply the first layer with ReLU activation + return tf.matmul(x,self.weight2)+self.bias2 # apply the second layer and return the output + + +class DDPG: # define a class for the DDPG agent + def __init__(self,hidden_dim,sigma,gamma,tau,actor_lr,critic_lr): # initialize the agent with hidden dimension, noise scale, discount factor, soft update factor, actor learning rate and critic learning rate + self.genv=gym.make('Pendulum-v0') # create environment + state_dim=self.genv.observation_space.shape[0] # get the state dimension from the environment observation space + action_dim=self.genv.action_space.shape[0] # get the action dimension from the environment action space + action_bound=self.genv.action_space.high[0] # get the action bound from the environment action space high value (assume symmetric) + self.actor=actor(state_dim,hidden_dim,action_dim,action_bound) # create an actor network for the agent + self.critic=critic(state_dim,hidden_dim,action_dim) # create a critic network for the agent + self.target_actor=actor(state_dim,hidden_dim,action_dim,action_bound) # create a target actor network for the agent + self.target_critic=critic(state_dim,hidden_dim,action_dim) # create a target critic network for the agent + self.actor_param=self.actor.param # parameter list of actor network, kernel uses it list for backpropagation + self.critic_param=self.critic.param # parameter list of critic network, kernel uses it list for backpropagation + self.target_actor.param=self.actor_param.copy() # copy the parameters from actor network to target actor network + self.target_critic.param=self.critic_param.copy() # copy the parameters from critic network to target critic network + self.param=[self.actor_param,self.critic_param] # parameter list of both networks, kernel uses it list for backpropagation + self.sigma=sigma # noise scale + self.gamma=gamma # discount factor + self.tau=tau # soft update factor + self.opt=tf.keras.optimizers.Adam() # optimizer, kernel uses it to optimize. Here we use Adam optimizer + + + def noise(self): # noise function, kernel uses it to generate exploration noise + return np.random.normal(scale=self.sigma) # return a random sample from a normal distribution with zero mean and sigma scale + + + def env(self,a=None,initial=None): # environment function, kernel uses it to interact with the environment + if initial==True: # if initial is True, reset the environment and return the initial state + state=self.genv.reset(seed=0) + return state + else: # otherwise, take an action and return the next state, reward and done flag + next_state,reward,done,_=self.genv.step(a) + return next_state,reward,done + + + def loss(self,s,a,next_s,r,d): # loss function, kernel uses it to calculate loss + a=tf.expand_dims(a,axis=1) # expand the action vector to match the Q-value matrix shape + next_q_value=self.target_critic.fp(next_s,self.target_actor.fp(next_s)) # get the Q-value for the next state and action from the target critic network + q_target=r+self.gamma*next_q_value*(1-d) # calculate the target value using Bellman equation with discount factor gamma + actor_loss=-tf.reduce_mean(self.critic.fp(s,self.actor.fp(s))) # calculate the actor loss as the negative mean of the Q-value for the current state and action from the critic network + critic_loss=tf.reduce_mean((self.critic.fp(s,a)-q_target)**2) # calculate the critic loss as the mean squared error between Q-value and target value + return [actor_loss,critic_loss] # return a list of actor loss and critic loss + + + def update_param(self): # update function, kernel uses it to update parameter + for target_param,param in zip(self.target_actor.param,self.actor.param): # for each pair of parameters in target actor network and actor network + target_param=target_param*(1.0-self.tau)+param*self.tau # update the target parameter using soft update with factor tau + for target_param,param in zip(self.target_critic.param,self.critic.param): # for each pair of parameters in target critic network and critic network + target_param=target_param*(1.0-self.tau)+param*self.tau # update the target parameter using soft update with factor tau + return \ No newline at end of file diff --git a/7.0/neuralnetwork/RL/tensorflow/non-parallel/DQN.py b/7.0/neuralnetwork/RL/tensorflow/non-parallel/DQN.py new file mode 100644 index 00000000..21487674 --- /dev/null +++ b/7.0/neuralnetwork/RL/tensorflow/non-parallel/DQN.py @@ -0,0 +1,46 @@ +import tensorflow as tf # import TensorFlow library +import gym # import OpenAI Gym library + + +class Qnet: # define a class for the Q-network + def __init__(self,state_dim,hidden_dim,action_dim): # initialize the network with state dimension, hidden dimension and action dimension + self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) # create a weight matrix for the first layer + self.bias1=tf.Variable(tf.random.normal([hidden_dim])) # create a bias vector for the first layer + self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) # create a weight matrix for the second layer + self.bias2=tf.Variable(tf.random.normal([action_dim])) # create a bias vector for the second layer + self.param=[self.weight1,self.bias1,self.weight2,self.bias2] # store the network parameters in a list + + def fp(self,x): # forward propagation function, kernel uses it for forward propagation + x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) # apply the first layer with ReLU activation + return tf.matmul(x,self.weight2)+self.bias2 # apply the second layer and return the output + + +class DQN: # define a class for the DQN agent + def __init__(self,state_dim,hidden_dim,action_dim): # initialize the agent with state dimension, hidden dimension and action dimension + self.nn=Qnet(state_dim,hidden_dim,action_dim) # create a Q-network for the agent + self.target_q_net=Qnet(state_dim,hidden_dim,action_dim) # create a target Q-network for the agent + self.param=self.nn.param # parameter list, kernel uses it list for backpropagation + self.opt=tf.keras.optimizers.Adam() # optimizer, kernel uses it to optimize + self.genv=gym.make('CartPole-v0') # create environment + + + def env(self,a=None,initial=None): # environment function, kernel uses it to interact with the environment + if initial==True: # if initial is True, reset the environment and return the initial state + state=self.genv.reset(seed=0) + return state + else: # otherwise, take an action and return the next state, reward and done flag + next_state,reward,done,_=self.genv.step(a) + return next_state,reward,done + + + def loss(self,s,a,next_s,r,d): # loss function, kernel uses it to calculate loss + a=tf.expand_dims(a,axis=1) # expand the action vector to match the Q-value matrix shape + q_value=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) # get the Q-value for the selected action + next_q_value=tf.reduce_max(self.target_q_net.fp(next_s),axis=1) # get the maximum Q-value for the next state from the target network + target=r+0.98*next_q_value*(1-d) # calculate the target value using Bellman equation with discount factor 0.98 + return tf.reduce_mean((q_value-target)**2) # return the mean squared error between Q-value and target value + + + def update_param(self): # update function, kernel uses it to update parameter + self.target_q_net.param=self.param.copy() # copy the parameters from the Q-network to the target network + return \ No newline at end of file diff --git a/7.0/neuralnetwork/RL/tensorflow/non-parallel/DQN_pr.py b/7.0/neuralnetwork/RL/tensorflow/non-parallel/DQN_pr.py new file mode 100644 index 00000000..77e8b8a9 --- /dev/null +++ b/7.0/neuralnetwork/RL/tensorflow/non-parallel/DQN_pr.py @@ -0,0 +1,59 @@ +import tensorflow as tf +import gym +import Note.create.RL.rl.prioritized_replay as pr + + +class Qnet: + def __init__(self,state_dim,hidden_dim,action_dim): + self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) + self.bias1=tf.Variable(tf.random.normal([hidden_dim])) + self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) + self.bias2=tf.Variable(tf.random.normal([action_dim])) + self.param=[self.weight1,self.bias1,self.weight2,self.bias2] + + def fp(self,x): + x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) + return tf.matmul(x,self.weight2)+self.bias2 + + +class DQN: + def __init__(self,state_dim,hidden_dim,action_dim): + self.nn=Qnet(state_dim,hidden_dim,action_dim) + self.target_q_net=Qnet(state_dim,hidden_dim,action_dim) + self.param=self.nn.param + self.pr=pr.pr() + self.initial_TD=7 + self._epsilon=0.0007 + self.alpha=0.7 + self.opt=tf.keras.optimizers.Adam() + self.genv=gym.make('CartPole-v0') + + + def env(self,a=None,initial=None): + if initial==True: + state=self.genv.reset(seed=0) + return state + else: + next_state,reward,done,_=self.genv.step(a) + return next_state,reward,done + + + def data_func(self,state_pool,action_pool,next_state_pool,reward_pool,done_pool,batch): + s,a,next_s,r,d=self.pr.sample(state_pool,action_pool,next_state_pool,reward_pool,done_pool,self.epsilon,self.alpha,batch) + return s,a,next_s,r,d + + + def loss(self,s,a,next_s,r,d): + a=tf.expand_dims(a,axis=1) + q_value=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) + next_q_value=tf.reduce_max(self.target_q_net.fp(next_s),axis=1) + target=r+0.98*next_q_value*(1-d) + target=tf.expand_dims(target,axis=1) + TD=target-q_value + self.pr.update_TD(TD) + return tf.reduce_mean(TD**2) + + + def update_param(self): + self.target_q_net.param=self.param.copy() + return \ No newline at end of file diff --git a/7.0/neuralnetwork/RL/tensorflow/parallel/DQN.py b/7.0/neuralnetwork/RL/tensorflow/parallel/DQN.py new file mode 100644 index 00000000..d7756475 --- /dev/null +++ b/7.0/neuralnetwork/RL/tensorflow/parallel/DQN.py @@ -0,0 +1,52 @@ +import tensorflow as tf # import TensorFlow library +import gym # import OpenAI Gym library +import Note.nn.parallel.optimizer as o # import Note's optimizer module + + +class Qnet: # define a class for the Q-network + def __init__(self,state_dim,hidden_dim,action_dim): # initialize the network with state dimension, hidden dimension and action dimension + self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) # create a weight matrix for the first layer + self.bias1=tf.Variable(tf.random.normal([hidden_dim])) # create a bias vector for the first layer + self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) # create a weight matrix for the second layer + self.bias2=tf.Variable(tf.random.normal([action_dim])) # create a bias vector for the second layer + self.param=[self.weight1,self.bias1,self.weight2,self.bias2] # store the network parameters in a list + + def fp(self,x): # forward propagation function, kernel uses it for forward propagation + x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) # apply the first layer with ReLU activation + return tf.matmul(x,self.weight2)+self.bias2 # apply the second layer and return the output + + +class DQN: # define a class for the DQN agent + def __init__(self,state_dim,hidden_dim,action_dim): # initialize the agent with state dimension, hidden dimension and action dimension + self.nn=Qnet(state_dim,hidden_dim,action_dim) # create a Q-network for the agent + self.target_q_net=Qnet(state_dim,hidden_dim,action_dim) # create a target Q-network for the agent + self.param=self.nn.param # parameter list, kernel uses it list for backpropagation + self.optimizer=o.Momentum(0.07,0.7) # optimizer, kernel uses it to optimize. Here we use a custom momentum optimizer with learning rate 0.07 and momentum 0.7 + self.genv=[gym.make('CartPole-v0') for _ in range(5)] # create a list of 5 environments + + + def env(self,a=None,p=None,initial=None): # environment function, kernel uses it to interact with the environment + if initial==True: # if initial is True, reset the environment with index p and return the initial state + state=self.genv[p].reset(seed=0) + return state + else: # otherwise, take an action and return the next state, reward and done flag from the environment with index p + next_state,reward,done,_=self.genv[p].step(a) + return next_state,reward,done + + + def loss(self,s,a,next_s,r,d): # loss function, kernel uses it to calculate loss + a=tf.expand_dims(a,axis=1) # expand the action vector to match the Q-value matrix shape + q_value=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) # get the Q-value for the selected action + next_q_value=tf.reduce_max(self.target_q_net.fp(next_s),axis=1) # get the maximum Q-value for the next state from the target network + target=r+0.98*next_q_value*(1-d) # calculate the target value using Bellman equation with discount factor 0.98 + return tf.reduce_mean((q_value-target)**2) # return the mean squared error between Q-value and target value + + + def update_param(self): # update function, kernel uses it to update parameter + self.target_q_net.param=self.param.copy() # copy the parameters from the Q-network to the target network + return + + + def opt(self,gradient): # optimization function, kernel uses it to optimize parameter + param=self.optimizer.opt(gradient,self.param) # apply the custom momentum optimizer to update the parameters using the gradient + return param \ No newline at end of file From d92bc748dde52dc98607de96fa7b029e52e07964 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Wed, 16 Aug 2023 14:21:24 +0800 Subject: [PATCH 176/337] Delete 7.0 directory --- .../DL/pytorch/non-parallel/nn.py | 53 --------- 7.0/neuralnetwork/DL/pytorch/parallel/nn.py | 41 ------- .../DL/pytorch/parallel/nn_device.py | 45 -------- 7.0/neuralnetwork/DL/tensorflow/layer/LSTM.py | 37 ------- .../DL/tensorflow/layer/attn_LSTM.py | 44 -------- 7.0/neuralnetwork/DL/tensorflow/layer/cnn.py | 50 --------- .../DL/tensorflow/layer/cnn_lmix.py | 58 ---------- 7.0/neuralnetwork/DL/tensorflow/layer/nn.py | 27 ----- .../DL/tensorflow/layer/nn_acc.py | 32 ------ .../DL/tensorflow/layer/nn_clipping.py | 40 ------- .../DL/tensorflow/layer/self_LSTM.py | 45 -------- .../DL/tensorflow/layer/transformer.py | 46 -------- .../DL/tensorflow/non-parallel/cnn.py | 27 ----- .../DL/tensorflow/non-parallel/lstm.py | 26 ----- .../DL/tensorflow/non-parallel/nn.py | 23 ---- .../DL/tensorflow/non-parallel/nn_acc.py | 28 ----- .../DL/tensorflow/non-parallel/nn_ol.py | 37 ------- .../DL/tensorflow/parallel/LSTM.py | 46 -------- .../DL/tensorflow/parallel/attn_LSTM.py | 53 --------- .../DL/tensorflow/parallel/cnn.py | 59 ---------- .../DL/tensorflow/parallel/cnn_lmix.py | 67 ------------ .../DL/tensorflow/parallel/nn.py | 39 ------- .../DL/tensorflow/parallel/nn_.py | 39 ------- .../DL/tensorflow/parallel/nn_acc.py | 44 -------- .../DL/tensorflow/parallel/nn_attenuate.py | 57 ---------- .../DL/tensorflow/parallel/nn_clipping.py | 47 -------- .../DL/tensorflow/parallel/nn_device.py | 40 ------- .../DL/tensorflow/parallel/self_LSTM.py | 54 ---------- .../DL/tensorflow/parallel/transformer.py | 55 ---------- .../RL/pytorch/non-parallel/DDPG.py | 101 ------------------ .../RL/pytorch/non-parallel/DQN.py | 64 ----------- .../RL/pytorch/non-parallel/DQN_pr.py | 76 ------------- .../RL/pytorch/non-parallel/DoubleDQN.py | 65 ----------- .../RL/pytorch/non-parallel/DuelingDQN.py | 67 ------------ 7.0/neuralnetwork/RL/pytorch/parallel/DQN.py | 65 ----------- .../RL/tensorflow/non-parallel/DDPG.py | 84 --------------- .../RL/tensorflow/non-parallel/DQN.py | 46 -------- .../RL/tensorflow/non-parallel/DQN_pr.py | 59 ---------- .../RL/tensorflow/parallel/DQN.py | 52 --------- 39 files changed, 1938 deletions(-) delete mode 100644 7.0/neuralnetwork/DL/pytorch/non-parallel/nn.py delete mode 100644 7.0/neuralnetwork/DL/pytorch/parallel/nn.py delete mode 100644 7.0/neuralnetwork/DL/pytorch/parallel/nn_device.py delete mode 100644 7.0/neuralnetwork/DL/tensorflow/layer/LSTM.py delete mode 100644 7.0/neuralnetwork/DL/tensorflow/layer/attn_LSTM.py delete mode 100644 7.0/neuralnetwork/DL/tensorflow/layer/cnn.py delete mode 100644 7.0/neuralnetwork/DL/tensorflow/layer/cnn_lmix.py delete mode 100644 7.0/neuralnetwork/DL/tensorflow/layer/nn.py delete mode 100644 7.0/neuralnetwork/DL/tensorflow/layer/nn_acc.py delete mode 100644 7.0/neuralnetwork/DL/tensorflow/layer/nn_clipping.py delete mode 100644 7.0/neuralnetwork/DL/tensorflow/layer/self_LSTM.py delete mode 100644 7.0/neuralnetwork/DL/tensorflow/layer/transformer.py delete mode 100644 7.0/neuralnetwork/DL/tensorflow/non-parallel/cnn.py delete mode 100644 7.0/neuralnetwork/DL/tensorflow/non-parallel/lstm.py delete mode 100644 7.0/neuralnetwork/DL/tensorflow/non-parallel/nn.py delete mode 100644 7.0/neuralnetwork/DL/tensorflow/non-parallel/nn_acc.py delete mode 100644 7.0/neuralnetwork/DL/tensorflow/non-parallel/nn_ol.py delete mode 100644 7.0/neuralnetwork/DL/tensorflow/parallel/LSTM.py delete mode 100644 7.0/neuralnetwork/DL/tensorflow/parallel/attn_LSTM.py delete mode 100644 7.0/neuralnetwork/DL/tensorflow/parallel/cnn.py delete mode 100644 7.0/neuralnetwork/DL/tensorflow/parallel/cnn_lmix.py delete mode 100644 7.0/neuralnetwork/DL/tensorflow/parallel/nn.py delete mode 100644 7.0/neuralnetwork/DL/tensorflow/parallel/nn_.py delete mode 100644 7.0/neuralnetwork/DL/tensorflow/parallel/nn_acc.py delete mode 100644 7.0/neuralnetwork/DL/tensorflow/parallel/nn_attenuate.py delete mode 100644 7.0/neuralnetwork/DL/tensorflow/parallel/nn_clipping.py delete mode 100644 7.0/neuralnetwork/DL/tensorflow/parallel/nn_device.py delete mode 100644 7.0/neuralnetwork/DL/tensorflow/parallel/self_LSTM.py delete mode 100644 7.0/neuralnetwork/DL/tensorflow/parallel/transformer.py delete mode 100644 7.0/neuralnetwork/RL/pytorch/non-parallel/DDPG.py delete mode 100644 7.0/neuralnetwork/RL/pytorch/non-parallel/DQN.py delete mode 100644 7.0/neuralnetwork/RL/pytorch/non-parallel/DQN_pr.py delete mode 100644 7.0/neuralnetwork/RL/pytorch/non-parallel/DoubleDQN.py delete mode 100644 7.0/neuralnetwork/RL/pytorch/non-parallel/DuelingDQN.py delete mode 100644 7.0/neuralnetwork/RL/pytorch/parallel/DQN.py delete mode 100644 7.0/neuralnetwork/RL/tensorflow/non-parallel/DDPG.py delete mode 100644 7.0/neuralnetwork/RL/tensorflow/non-parallel/DQN.py delete mode 100644 7.0/neuralnetwork/RL/tensorflow/non-parallel/DQN_pr.py delete mode 100644 7.0/neuralnetwork/RL/tensorflow/parallel/DQN.py diff --git a/7.0/neuralnetwork/DL/pytorch/non-parallel/nn.py b/7.0/neuralnetwork/DL/pytorch/non-parallel/nn.py deleted file mode 100644 index 1faec0fc..00000000 --- a/7.0/neuralnetwork/DL/pytorch/non-parallel/nn.py +++ /dev/null @@ -1,53 +0,0 @@ -import torch -from torch import nn - - -class NeuralNetwork(nn.Module): - def __init__(self): - super().__init__() - self.flatten=nn.Flatten() - self.linear_relu_stack=nn.Sequential( - nn.Linear(28*28, 512), - nn.ReLU(), - nn.Linear(512, 512), - nn.ReLU(), - nn.Linear(512, 10) - ) - - - def forward(self,x): - x=self.flatten(x) - logits=self.linear_relu_stack(x) - return logits - - -class neuralnetwork: - def __init__(self): - if torch.cuda.is_available(): - self.device=torch.device('cuda') - else: - self.device=torch.device('cpu') - self.model=NeuralNetwork().to(self.device) - self.loss_fn=nn.CrossEntropyLoss() - self.opt=torch.optim.SGD(self.model.parameters(),lr=1e-3) #optimizer,kernel uses it to optimize. - - - def fp(self,x): #forward propagation function,kernel uses it for forward propagation. - pred=self.model(x.to(self.device)) - return pred - - - def loss(self,output,labels): #loss functino,kernel uses it to calculate loss. - loss=self.loss_fn(output,labels.to(self.device)) - return loss - - - def backward(self,loss): #backward function,kernel uses it for backpropagation. - self.optim.zero_grad() - loss.backward() - return - - - def opt(self): #opt function,kernel uses it to optimize. - self.optim.step() - return \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/pytorch/parallel/nn.py b/7.0/neuralnetwork/DL/pytorch/parallel/nn.py deleted file mode 100644 index d9b05567..00000000 --- a/7.0/neuralnetwork/DL/pytorch/parallel/nn.py +++ /dev/null @@ -1,41 +0,0 @@ -import torch # import the PyTorch library -from torch import nn # import the neural network module from PyTorch - -class NeuralNetwork(nn.Module): # define a class for the neural network model - def __init__(self): # define the constructor method - super().__init__() # call the parent class constructor - self.flatten = nn.Flatten() # define a layer to flatten the input - self.linear_relu_stack=nn.Sequential( # define a stack of linear and relu layers - nn.Linear(28*28,512), # a linear layer with 28*28 input features and 512 output features - nn.ReLU(), # a relu activation function - nn.Linear(512,512), # another linear layer with 512 input and output features - nn.ReLU(), # another relu activation function - nn.Linear(512,10) # a final linear layer with 512 input features and 10 output features - ) - - - def forward(self,x): # define the forward method - x = self.flatten(x) # flatten the input x - logits=self.linear_relu_stack(x) # pass x through the linear relu stack and get the logits - return logits # return the logits - - -class neuralnetwork: # define another class for the neural network object - def __init__(self): # define the constructor method - if torch.cuda.is_available(): # check if cuda is available - self.device=torch.device('cuda') # set the device to cuda - else: - self.device=torch.device('cpu') # otherwise set the device to cpu - self.model=NeuralNetwork().to(self.device) # create an instance of the NeuralNetwork class and move it to the device - self.loss_fn=nn.CrossEntropyLoss() # define the loss function as cross entropy loss - self.opt=[torch.optim.SGD(self.model.parameters(),lr=1e-3) for _ in range(7)] # define a list of three stochastic gradient descent optimizers with learning rate 1e-3 - - - def fp(self,x): # define a method for forward propagation - pred=self.model(x.to(self.device)) # get the predictions from the model by passing x to the device - return pred # return the predictions - - - def loss(self,output,labels): # define a method for calculating the loss - loss=self.loss_fn(output,labels.to(self.device)) # compute the loss by passing output and labels to the device - return loss # return the loss \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/pytorch/parallel/nn_device.py b/7.0/neuralnetwork/DL/pytorch/parallel/nn_device.py deleted file mode 100644 index ad4b8069..00000000 --- a/7.0/neuralnetwork/DL/pytorch/parallel/nn_device.py +++ /dev/null @@ -1,45 +0,0 @@ -import torch # import the PyTorch library -from torch import nn # import the neural network module from PyTorch -from Note.nn.parallel.assign_device_pytorch import assign_device # import a custom function to assign device for PyTorch tensors - - -class NeuralNetwork(nn.Module): # define a class for the neural network model - def __init__(self): # define the constructor method - super().__init__() # call the parent class constructor - self.flatten = nn.Flatten() # define a layer to flatten the input tensor - self.linear_relu_stack=nn.Sequential( # define a stack of linear and relu layers - nn.Linear(28*28,512), # define a linear layer with input size 28*28 and output size 512 - nn.ReLU(), # define a relu activation function - nn.Linear(512,512), # define another linear layer with input and output size 512 - nn.ReLU(), # define another relu activation function - nn.Linear(512,10) # define the final linear layer with output size 10 - ) - - - def forward(self,x): # define the forward method for the model - x = self.flatten(x) # flatten the input tensor - logits=self.linear_relu_stack(x) # pass the flattened tensor through the linear relu stack - return logits # return the logits as the output - - -class neuralnetwork: # define another class for the neural network object - def __init__(self): # define the constructor method - if torch.cuda.is_available(): # check if cuda device is available - self.device=torch.device('cuda') # set the device to cuda - else: # otherwise - self.device=torch.device('cpu') # set the device to cpu - self.model=NeuralNetwork().to(self.device) # create an instance of the neural network model and move it to the device - self.loss_fn=nn.CrossEntropyLoss() # define the loss function as cross entropy loss - self.opt=[torch.optim.SGD(self.model.parameters(),lr=1e-3) for _ in range(7)] # define a list of optimizers as stochastic gradient descent with learning rate 1e-3 - - - def fp(self,x,p): # define a method for forward propagation - pred=self.model(x.to(assign_device(p,'GPU'))) # assign the device according to the process index p - # get the predictions from the model by passing the input tensor to the device and then to the model - return pred # return the predictions - - - def loss(self,output,labels,p): # define a method for calculating loss - loss=self.loss_fn(output,labels.to(assign_device(p,'GPU'))) # assign the device according to the process index p - # calculate the loss by passing the output and labels tensors to the device and then to the loss function - return loss # return the loss \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/layer/LSTM.py b/7.0/neuralnetwork/DL/tensorflow/layer/LSTM.py deleted file mode 100644 index 272855a3..00000000 --- a/7.0/neuralnetwork/DL/tensorflow/layer/LSTM.py +++ /dev/null @@ -1,37 +0,0 @@ -import tensorflow as tf -from Note.nn.layer.LSTM import LSTM -from Note.nn.layer.dense import dense - -# Define a recurrent neural network class with LSTM layers -class lstm: - def __init__(self): - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.opt=tf.keras.optimizers.Adam() - - - def build(self): - # Create two LSTM layers with 50 hidden units each - self.lstm1=LSTM(weight_shape=(3,50),return_sequence=True) # First LSTM layer that returns the full sequence - self.lstm2=LSTM(weight_shape=(50,50),) # Second LSTM layer that returns the last output only - # Create a dense layer with 10 output units - self.dense=dense([50, 10]) - # Store the parameters of the layers in a list - self.param=[self.lstm1.param, - self.lstm2.param, - self.dense.param, - ] - return - - - def fp(self,data): - # Perform forward propagation on the input data - x=self.lstm1.output(data) # First LSTM layer output - x=self.lstm2.output(x) # Second LSTM layer output - output=self.dense.output(x) # Dense layer output - return output - - - def loss(self, output, labels): - # Compute the loss between the output and the labels - return self.loss_object(labels, output) \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/layer/attn_LSTM.py b/7.0/neuralnetwork/DL/tensorflow/layer/attn_LSTM.py deleted file mode 100644 index 1d41befd..00000000 --- a/7.0/neuralnetwork/DL/tensorflow/layer/attn_LSTM.py +++ /dev/null @@ -1,44 +0,0 @@ -import tensorflow as tf -from Note.nn.layer.LSTM import LSTM -from Note.nn.layer.dense import dense -from Note.nn.layer.attention import attention - -# Define a recurrent neural network class with LSTM layers and attention -class attn_lstm: - def __init__(self): - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.opt=tf.keras.optimizers.Adam() - - - def build(self): - # Create two LSTM layers with 50 hidden units each - self.lstm1=LSTM(weight_shape=(3,50),return_sequence=True) # First LSTM layer that returns the full sequence - self.lstm2=LSTM(weight_shape=(50,50),) # Second LSTM layer that returns the last output only - # Create a dense layer with 10 output units and softmax activation - self.dense=dense([100, 10],activation='softmax') - # Create an attention object with weight shape (50, 50) - self.attn=attention((50,50)) - # Store the parameters of the layers and the attention in a list - self.param=[self.lstm1.param, - self.lstm2.param, - self.dense.param, - self.attn.param] # Add the attention parameters to the list - return - - - def fp(self,data): - # Perform forward propagation on the input data - x=self.lstm1.output(data) # First LSTM layer output, shape (B, N, 50) - y=self.lstm2.output(x) # Second LSTM layer output, shape (B, 50) - # Use the attention object to compute the context vector from x and y, shape (B, 50) - context_vector,_,_=self.attn.output(x,y) - # Concatenate the context vector and the second LSTM output, shape (B, 100) - z=tf.concat([context_vector,y],axis=-1) - output=self.dense.output(z) # Dense layer output with softmax activation, shape (B, 10) - return output - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/layer/cnn.py b/7.0/neuralnetwork/DL/tensorflow/layer/cnn.py deleted file mode 100644 index aa2667f8..00000000 --- a/7.0/neuralnetwork/DL/tensorflow/layer/cnn.py +++ /dev/null @@ -1,50 +0,0 @@ -import tensorflow as tf -from Note.nn.layer.conv2d import conv2d -from Note.nn.layer.dense import dense -from Note.nn.layer.flatten import flatten - -# Define a convolutional neural network class -class cnn: - def __init__(self): - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.opt=tf.keras.optimizers.Adam() - - - def build(self): - # Create three convolutional layers with relu activations - self.conv1=conv2d(weight_shape=(3,3,3,32),activation='relu') - self.conv2=conv2d(weight_shape=(3,3,32,64),activation='relu') - self.conv3=conv2d(weight_shape=(3,3,64,64),activation='relu') - # Create two dense layers with relu and linear activations - self.dense1=dense([64*4*4,64],activation='relu') - self.dense2=dense([64,10]) - # Store the parameters of the layers in a list - self.param = [self.conv1.weight, - self.conv2.weight, - self.conv3.weight, - self.dense1.weight, - self.dense1.bias, - self.dense2.weight, - self.dense2.bias] - return - - - def fp(self,data): - # Perform forward propagation on the input data - x=tf.nn.conv2d(data,self.conv1.weight,strides=(1,1),padding='SAME') # First convolutional layer - x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # First max pooling layer - x=tf.nn.conv2d(x,self.conv2.weight,strides=(1,1),padding='SAME') # Second convolutional layer - x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # Second max pooling layer - x=tf.nn.conv2d(x,self.conv3.weight,strides=(1,1),padding='SAME') # Third convolutional layer - x=flatten(x) # Flatten the output to a vector - x=tf.nn.relu(tf.matmul(x,self.dense1.weight)+self.dense1.bias) # First dense layer with relu activation - x=tf.nn.dropout(x,rate=0.5) # Apply dropout to prevent overfitting - output=tf.matmul(x,self.dense2.weight)+self.dense2.bias # Output layer with linear activation - output=tf.nn.dropout(output,rate=0.5) # Apply dropout to prevent overfitting - return output - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/layer/cnn_lmix.py b/7.0/neuralnetwork/DL/tensorflow/layer/cnn_lmix.py deleted file mode 100644 index 40232919..00000000 --- a/7.0/neuralnetwork/DL/tensorflow/layer/cnn_lmix.py +++ /dev/null @@ -1,58 +0,0 @@ -import tensorflow as tf -from Note.nn.layer.conv2d import conv2d -from Note.nn.layer.dense import dense -from Note.nn.layer.flatten import flatten -from Note.nn.layer.LMix import lmix - -# Define a convolutional neural network class with mixup augmentation -class cnn: - def __init__(self): - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.opt=tf.keras.optimizers.Adam() - # Initialize the alpha parameter for mixup - self.alpha=1.0 - - - def build(self): - # Create three convolutional layers with relu activations - self.conv1=conv2d(weight_shape=(3,3,3,32),activation='relu') - self.conv2=conv2d(weight_shape=(3,3,32,64),activation='relu') - self.conv3=conv2d(weight_shape=(3,3,64,64),activation='relu') - # Create two dense layers with relu and linear activations - self.dense1=dense([64*4*4,64],activation='relu') - self.dense2=dense([64,10]) - # Store the parameters of the layers in a list - self.param = [self.conv1.weight, - self.conv2.weight, - self.conv3.weight, - self.dense1.weight, - self.dense1.bias, - self.dense2.weight, - self.dense2.bias] - return - - - def data_func(self,data_batch,labels_batch): - # Apply mixup augmentation on the input data and labels - return lmix(data_batch,labels_batch,self.alpha) - - - def fp(self,data): - # Perform forward propagation on the input data - x=tf.nn.conv2d(data,self.conv1.weight,strides=(1,1),padding='SAME') # First convolutional layer - x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # First max pooling layer - x=tf.nn.conv2d(x,self.conv2.weight,strides=(1,1),padding='SAME') # Second convolutional layer - x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # Second max pooling layer - x=tf.nn.conv2d(x,self.conv3.weight,strides=(1,1),padding='SAME') # Third convolutional layer - x=flatten(x) # Flatten the output to a vector - x=tf.nn.relu(tf.matmul(x,self.dense1.weight)+self.dense1.bias) # First dense layer with relu activation - x=tf.nn.dropout(x,rate=0.5) # Apply dropout to prevent overfitting - output=tf.matmul(x,self.dense2.weight)+self.dense2.bias # Output layer with linear activation - output=tf.nn.dropout(output,rate=0.5) # Apply dropout to prevent overfitting - return output - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/layer/nn.py b/7.0/neuralnetwork/DL/tensorflow/layer/nn.py deleted file mode 100644 index 77ed7454..00000000 --- a/7.0/neuralnetwork/DL/tensorflow/layer/nn.py +++ /dev/null @@ -1,27 +0,0 @@ -import tensorflow as tf -import Note.nn.layer.dense as d -from Note.nn.layer.flatten import flatten - - -class nn: - def __init__(self): - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.opt=tf.keras.optimizers.Adam() - - - def build(self): - self.layer1=d.dense([784,128],activation='relu') - self.layer2=d.dense([128,10]) - self.param=[self.layer1.param,self.layer2.param] - return - - - def fp(self,data): - data=flatten(data) - output1=self.layer1.output(data) - output2=self.layer2.output(output1) - return output2 - - - def loss(self,output,labels): - return self.loss_object(labels,output) \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/layer/nn_acc.py b/7.0/neuralnetwork/DL/tensorflow/layer/nn_acc.py deleted file mode 100644 index cad20961..00000000 --- a/7.0/neuralnetwork/DL/tensorflow/layer/nn_acc.py +++ /dev/null @@ -1,32 +0,0 @@ -import tensorflow as tf -import Note.nn.layer.dense as d -from Note.nn.layer.flatten import flatten - - -class nn: - def __init__(self): - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.train_accuracy=tf.keras.metrics.SparseCategoricalAccuracy(name='train_accuracy') - self.opt=tf.keras.optimizers.Adam() - - - def build(self): - self.layer1=d.dense([784,128],activation='relu') - self.layer2=d.dense([128,10]) - self.param=[self.layer1.param,self.layer2.param] - return - - - def fp(self,data): - data=flatten(data) - output1=self.layer1.output(data) - output2=self.layer2.output(output1) - return output2 - - - def loss(self,output,labels): - return self.loss_object(labels,output) - - - def accuracy(self,output,labels): - return self.train_accuracy(labels,output) \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/layer/nn_clipping.py b/7.0/neuralnetwork/DL/tensorflow/layer/nn_clipping.py deleted file mode 100644 index b53a43da..00000000 --- a/7.0/neuralnetwork/DL/tensorflow/layer/nn_clipping.py +++ /dev/null @@ -1,40 +0,0 @@ -import tensorflow as tf -import Note.nn.layer.dense as d -from Note.nn.layer.flatten import flatten -from Note.nn.gradient_clipping import gradient_clipping - -# Define a neural network class -class nn: - def __init__(self): - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.opt=tf.keras.optimizers.Adam() - - - def build(self): - # Create two dense layers with relu and linear activations - self.layer1=d.dense([784,128],activation='relu') - self.layer2=d.dense([128,10]) - # Store the parameters of the layers in a list - self.param=[self.layer1.param,self.layer2.param] - return - - - def fp(self,data): - # Perform forward propagation on the input data - data=flatten(data) - output1=self.layer1.output(data) - output2=self.layer2.output(output1) - return output2 - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) - - - def gradient(self,tape,loss): - # Compute the gradients of the loss with respect to the parameters - grads=tape.gradient(loss,self.param) - # Clip the gradients by value to avoid exploding gradients - return gradient_clipping(grads,'value',1.0) \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/layer/self_LSTM.py b/7.0/neuralnetwork/DL/tensorflow/layer/self_LSTM.py deleted file mode 100644 index edaa2420..00000000 --- a/7.0/neuralnetwork/DL/tensorflow/layer/self_LSTM.py +++ /dev/null @@ -1,45 +0,0 @@ -import tensorflow as tf -from Note.nn.layer.LSTM import LSTM -from Note.nn.layer.self_attention import self_attention -from Note.nn.layer.dense import dense -from Note.nn.layer.emdedding import embedding - -# Define a LSTM self attention neural network class -class self_LSTM: - def __init__(self,vocab_size,embed_size,num_heads,num_layers): - self.vocab_size=vocab_size - self.embed_size=embed_size - self.num_heads=num_heads - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.opt=tf.keras.optimizers.Adam() - - - def build(self): - # Create an embedding layer to map input tokens to vectors - self.embedding_layer = embedding(self.vocab_size,self.embed_size) - # Create a LSTM layer to process the input sequence - self.lstm_layer = LSTM(weight_shape=[self.embed_size,self.embed_size],return_sequence=True) - # Create a self attention layer to attend to the LSTM output - self.self_attention_layer = self_attention(weight_shape=[self.embed_size,self.embed_size],num_heads=self.num_heads) - # Create a linear layer to map the attention output to logits - self.logits_layer = dense([self.embed_size,self.vocab_size]) - # Store the parameters of the layers in a list - self.param = [self.embedding_layer.embeddings] - self.param.extend(self.lstm_layer.param) - self.param.extend(self.self_attention_layer.param) - self.param.extend(self.logits_layer.param) - - - def fp(self,data): - # Perform forward propagation on the input data - x = self.embedding_layer.output(data) # shape: (batch_size, seq_len, embed_size) - x = self.lstm_layer.output(x) # shape: (batch_size, seq_len, embed_size) - x, attention_weights = self.self_attention_layer.output(x,self.num_heads) # shape: (batch_size, seq_len, embed_size) - output = self.logits_layer.output(x) # shape: (batch_size, seq_len, vocab_size) - return output - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/layer/transformer.py b/7.0/neuralnetwork/DL/tensorflow/layer/transformer.py deleted file mode 100644 index 47862006..00000000 --- a/7.0/neuralnetwork/DL/tensorflow/layer/transformer.py +++ /dev/null @@ -1,46 +0,0 @@ -import tensorflow as tf -from Note.nn.layer.transformer import transformer -from Note.nn.layer.dense import dense -from Note.nn.layer.emdedding import embedding -from Note.nn.positional_encoding import positional_encoding - -# Define a positional encoding transformer neural network class -class Transformer: - def __init__(self,vocab_size,embed_size,num_heads,num_layers,max_len): - self.vocab_size=vocab_size - self.embed_size=embed_size - self.num_heads=num_heads - self.num_layers=num_layers - self.max_len=max_len - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.opt=tf.keras.optimizers.Adam() - - - def build(self): - # Create an embedding layer to map input tokens to vectors - self.embedding_layer = embedding(self.vocab_size,self.embed_size) - # Create a list of transformer layers - self.transformer_layers = [transformer(weight_shape=[self.embed_size, self.embed_size], num_heads=self.num_heads) for _ in range(self.num_layers)] - # Create a linear layer to map the transformer output to logits - self.logits_layer = dense([self.embed_size,self.vocab_size]) - # Store the parameters of the layers in a list - self.param = [self.embedding_layer.embeddings] - for transformer_layer in self.transformer_layers: - self.param.extend(transformer_layer.param) - self.param.extend(self.logits_layer.param) - - - def fp(self,data): - # Perform forward propagation on the input data - x = self.embedding_layer.output(data) # shape: (batch_size, seq_len, embed_size) - x += positional_encoding(self.max_len,self.embed_size) # shape: (batch_size, seq_len, embed_size) - for transformer_layer in self.transformer_layers: - x = transformer_layer.output(x) # shape: (batch_size, seq_len, embed_size) - output = self.logits_layer.output(x) # shape: (batch_size, seq_len, vocab_size) - return output - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/non-parallel/cnn.py b/7.0/neuralnetwork/DL/tensorflow/non-parallel/cnn.py deleted file mode 100644 index bd7ad1bf..00000000 --- a/7.0/neuralnetwork/DL/tensorflow/non-parallel/cnn.py +++ /dev/null @@ -1,27 +0,0 @@ -import tensorflow as tf -from tensorflow.keras import layers,models - - -class cnn: - def __init__(self): - self.model=models.Sequential() - self.model.add(layers.Conv2D(32,(3,3),activation='relu',input_shape=(32,32,3))) - self.model.add(layers.MaxPooling2D((2,2))) - self.model.add(layers.Conv2D(64,(3,3),activation='relu')) - self.model.add(layers.MaxPooling2D((2,2))) - self.model.add(layers.Conv2D(64,(3,3),activation='relu')) - self.model.add(layers.Flatten()) - self.model.add(layers.Dense(64,activation='relu')) - self.model.add(layers.Dense(10)) - self.param=self.model.weights - self.opt=tf.keras.optimizers.Adam() - - - def fp(self,data): - output=self.model(data) - return output - - - def loss(self,output,labels): - loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - return loss(labels,output) \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/non-parallel/lstm.py b/7.0/neuralnetwork/DL/tensorflow/non-parallel/lstm.py deleted file mode 100644 index 29b9b396..00000000 --- a/7.0/neuralnetwork/DL/tensorflow/non-parallel/lstm.py +++ /dev/null @@ -1,26 +0,0 @@ -import tensorflow as tf - - -class lstm: - def __init__(self,encoder): - self.model=tf.keras.Sequential([ - encoder, - tf.keras.layers.Embedding(len(encoder.get_vocabulary()), 64, mask_zero=True), - tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(64, return_sequences=True)), - tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(32)), - tf.keras.layers.Dense(64, activation='relu'), - tf.keras.layers.Dropout(0.5), - tf.keras.layers.Dense(1) - ]) - self.param=self.model.weights[1:] - self.opt=tf.keras.optimizers.Adam() - - - def fp(self,data): - output=self.model(data) - return output - - - def loss(self,output,labels): - bce=tf.keras.losses.BinaryCrossentropy(from_logits=True) - return bce(labels,output) \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/non-parallel/nn.py b/7.0/neuralnetwork/DL/tensorflow/non-parallel/nn.py deleted file mode 100644 index b4f03a92..00000000 --- a/7.0/neuralnetwork/DL/tensorflow/non-parallel/nn.py +++ /dev/null @@ -1,23 +0,0 @@ -import tensorflow as tf - - -class nn: - def __init__(self): - self.model=tf.keras.models.Sequential([ - tf.keras.layers.Flatten(input_shape=(28, 28)), - tf.keras.layers.Dense(128,activation='relu'), - tf.keras.layers.Dropout(0.2), - tf.keras.layers.Dense(10) - ]) - self.param=self.model.weights #parameter list,kernel uses it list for backpropagation. - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.opt=tf.keras.optimizers.Adam() #optimizer,kernel uses it to optimize. - - - def fp(self,data): #forward propagation function,kernel uses it for forward propagation. - output=self.model(data) - return output - - - def loss(self,output,labels): #loss functino,kernel uses it to calculate loss. - return self.loss_object(labels,output) \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/non-parallel/nn_acc.py b/7.0/neuralnetwork/DL/tensorflow/non-parallel/nn_acc.py deleted file mode 100644 index 0a726603..00000000 --- a/7.0/neuralnetwork/DL/tensorflow/non-parallel/nn_acc.py +++ /dev/null @@ -1,28 +0,0 @@ -import tensorflow as tf - -#An example with accuracy function. -class nn: - def __init__(self): - self.model=tf.keras.models.Sequential([ - tf.keras.layers.Flatten(input_shape=(28, 28)), - tf.keras.layers.Dense(128,activation='relu'), - tf.keras.layers.Dropout(0.2), - tf.keras.layers.Dense(10) - ]) - self.param=self.model.weights #parameter list,kernel uses it list for backpropagation. - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.train_accuracy=tf.keras.metrics.SparseCategoricalAccuracy(name='train_accuracy') - self.opt=tf.keras.optimizers.Adam() #optimizer,kernel uses it to optimize. - - - def fp(self,data): #forward propagation function,kernel uses it for forward propagation. - output=self.model(data) - return output - - - def loss(self,output,labels): #loss functino,kernel uses it to calculate loss. - return self.loss_object(labels,output) - - - def accuracy(self,output,labels): #accuracy function,kernel uses it to calculate accuracy. - return self.train_accuracy(labels,output) \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/non-parallel/nn_ol.py b/7.0/neuralnetwork/DL/tensorflow/non-parallel/nn_ol.py deleted file mode 100644 index 9dfa608d..00000000 --- a/7.0/neuralnetwork/DL/tensorflow/non-parallel/nn_ol.py +++ /dev/null @@ -1,37 +0,0 @@ -import tensorflow as tf # import TensorFlow library -import numpy as np # import NumPy library - -# This is an example of online training. -class nn: # define a class for the neural network - def __init__(self,data,labels): # initialize the network with data and labels - self.train_data=data # store the training data - self.train_labels=labels # store the training labels - self.model=tf.keras.models.Sequential([ # create a sequential model with four layers - tf.keras.layers.Flatten(input_shape=(28, 28)), # flatten the input to a vector of 28*28 elements - tf.keras.layers.Dense(128,activation='relu'), # add a dense layer with 128 units and ReLU activation - tf.keras.layers.Dropout(0.2), # add a dropout layer with 0.2 dropout rate to prevent overfitting - tf.keras.layers.Dense(10) # add a dense layer with 10 units for the output logits - ]) - self.param=self.model.weights # parameter list, kernel uses it list for backpropagation - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) # loss object, kernel uses it to calculate loss. Here we use sparse categorical crossentropy with logits as output - self.opt=tf.keras.optimizers.Adam() # optimizer, kernel uses it to optimize. Here we use Adam optimizer - self.train_loss_list=[] # a list to store the training loss values - self.counter=0 # counter to keep track of the number of online updates - self.max_length=1000 # maximum length of train_loss_list - - - def online(self): # This is simulative online function, kernel uses it to get online data and labels - index=np.random.choice(60000,size=[32]) # randomly sample 32 indices from 0 to 59999 - if self.counter==10000: # if the counter reaches 10000, stop the online training - return 'stop' - else: # otherwise, return the data and labels corresponding to the sampled indices - return [self.train_data[index],self.train_labels[index]] - - - def fp(self,data): # forward propagation function, kernel uses it for forward propagation - output=self.model(data) # pass the data through the model and get the output logits - return output - - - def loss(self,output,labels): # loss function, kernel uses it to calculate loss - return self.loss_object(labels,output) # return the sparse categorical crossentropy loss between labels and output \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/LSTM.py b/7.0/neuralnetwork/DL/tensorflow/parallel/LSTM.py deleted file mode 100644 index e8686c10..00000000 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/LSTM.py +++ /dev/null @@ -1,46 +0,0 @@ -import tensorflow as tf -from Note.nn.layer.LSTM import LSTM -from Note.nn.layer.dense import dense -from Note.nn.parallel.optimizer import Adam - -# Define a recurrent neural network class with LSTM layers -class lstm: - def __init__(self): - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.optimizer=Adam() - # Initialize a variable to keep track of the batch count - self.bc=tf.Variable(0, dtype=tf.float32) - - - def build(self): - # Create two LSTM layers with 50 hidden units each - self.lstm1=LSTM(weight_shape=(3,50),return_sequence=True) # First LSTM layer that returns the full sequence - self.lstm2=LSTM(weight_shape=(50,50),) # Second LSTM layer that returns the last output only - # Create a dense layer with 10 output units - self.dense=dense([50, 10]) - # Store the parameters of the layers in a list - self.param=[self.lstm1.param, - self.lstm2.param, - self.dense.param, - ] - return - - - def fp(self,data): - # Perform forward propagation on the input data - x=self.lstm1.output(data) # First LSTM layer output - x=self.lstm2.output(x) # Second LSTM layer output - output=self.dense.output(x) # Dense layer output - return output - - - def loss(self, output, labels): - # Compute the loss between the output and the labels - return self.loss_object(labels, output) - - - def opt(self,gradient): - # Perform optimization on the parameters using the gradient and the batch count - param=self.optimizer.opt(gradient,self.param,self.bc[0]) - return param \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/attn_LSTM.py b/7.0/neuralnetwork/DL/tensorflow/parallel/attn_LSTM.py deleted file mode 100644 index 198e46b1..00000000 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/attn_LSTM.py +++ /dev/null @@ -1,53 +0,0 @@ -import tensorflow as tf -from Note.nn.layer.LSTM import LSTM -from Note.nn.layer.dense import dense -from Note.nn.layer.attention import attention -from Note.nn.parallel.optimizer import Adam - -# Define a recurrent neural network class with LSTM layers and attention -class attn_lstm: - def __init__(self): - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.optimizer=Adam() - # Initialize a variable to keep track of the batch count - self.bc=tf.Variable(0, dtype=tf.float32) - - - def build(self): - # Create two LSTM layers with 50 hidden units each - self.lstm1=LSTM(weight_shape=(3,50),return_sequence=True) # First LSTM layer that returns the full sequence - self.lstm2=LSTM(weight_shape=(50,50),) # Second LSTM layer that returns the last output only - # Create a dense layer with 10 output units and softmax activation - self.dense=dense([100, 10],activation='softmax') - # Create an attention object with weight shape (50, 50) - self.attn=attention((50,50)) - # Store the parameters of the layers and the attention in a list - self.param=[self.lstm1.param, - self.lstm2.param, - self.dense.param, - self.attn.param] # Add the attention parameters to the list - return - - - def fp(self,data): - # Perform forward propagation on the input data - x=self.lstm1.output(data) # First LSTM layer output, shape (B, N, 50) - y=self.lstm2.output(x) # Second LSTM layer output, shape (B, 50) - # Use the attention object to compute the context vector from x and y, shape (B, 50) - context_vector,_,_=self.attn.output(x,y) - # Concatenate the context vector and the second LSTM output, shape (B, 100) - z=tf.concat([context_vector,y],axis=-1) - output=self.dense.output(z) # Dense layer output with softmax activation, shape (B, 10) - return output - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) - - - def opt(self,gradient): - # Perform optimization on the parameters using the gradient and the batch count - param=self.optimizer.opt(gradient,self.param,self.bc[0]) - return param \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/cnn.py b/7.0/neuralnetwork/DL/tensorflow/parallel/cnn.py deleted file mode 100644 index 59be2e63..00000000 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/cnn.py +++ /dev/null @@ -1,59 +0,0 @@ -import tensorflow as tf -from Note.nn.layer.conv2d import conv2d -from Note.nn.layer.dense import dense -from Note.nn.layer.flatten import flatten -from Note.nn.parallel.optimizer import Adam - -# Define a convolutional neural network class -class cnn: - def __init__(self): - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.optimizer=Adam() - # Initialize a variable to keep track of the batch count - self.bc=tf.Variable(0,dtype=tf.float32) - - - def build(self): - # Create three convolutional layers with relu activations - self.conv1=conv2d(weight_shape=(3,3,3,32),activation='relu') - self.conv2=conv2d(weight_shape=(3,3,32,64),activation='relu') - self.conv3=conv2d(weight_shape=(3,3,64,64),activation='relu') - # Create two dense layers with relu and linear activations - self.dense1=dense([64*4*4,64],activation='relu') - self.dense2=dense([64,10]) - # Store the parameters of the layers in a list - self.param = [self.conv1.weight, - self.conv2.weight, - self.conv3.weight, - self.dense1.weight, - self.dense1.bias, - self.dense2.weight, - self.dense2.bias] - return - - - def fp(self,data): - # Perform forward propagation on the input data - x=tf.nn.conv2d(data,self.conv1.weight,strides=(1,1),padding='SAME') # First convolutional layer - x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # First max pooling layer - x=tf.nn.conv2d(x,self.conv2.weight,strides=(1,1),padding='SAME') # Second convolutional layer - x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # Second max pooling layer - x=tf.nn.conv2d(x,self.conv3.weight,strides=(1,1),padding='SAME') # Third convolutional layer - x=flatten(x) # Flatten the output to a vector - x=tf.nn.relu(tf.matmul(x,self.dense1.weight)+self.dense1.bias) # First dense layer with relu activation - x=tf.nn.dropout(x,rate=0.5) # Apply dropout to prevent overfitting - output=tf.matmul(x,self.dense2.weight)+self.dense2.bias # Output layer with linear activation - output=tf.nn.dropout(output,rate=0.5) # Apply dropout to prevent overfitting - return output - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) - - - def opt(self,gradient): - # Perform optimization on the parameters using the gradient and the batch count - param=self.optimizer.opt(gradient,self.param,self.bc[0]) - return param \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/cnn_lmix.py b/7.0/neuralnetwork/DL/tensorflow/parallel/cnn_lmix.py deleted file mode 100644 index b6e66458..00000000 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/cnn_lmix.py +++ /dev/null @@ -1,67 +0,0 @@ -import tensorflow as tf -from Note.nn.layer.conv2d import conv2d -from Note.nn.layer.dense import dense -from Note.nn.layer.flatten import flatten -from Note.nn.parallel.optimizer import Adam -from Note.nn.layer.LMix import lmix - -# Define a convolutional neural network class with mixup augmentation -class cnn: - def __init__(self): - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.optimizer=Adam() - # Initialize the alpha parameter for mixup - self.alpha=1.0 - # Initialize a variable to keep track of the batch count - self.bc=tf.Variable(0,dtype=tf.float32) - - - def build(self): - # Create three convolutional layers with relu activations - self.conv1=conv2d(weight_shape=(3,3,3,32),activation='relu') - self.conv2=conv2d(weight_shape=(3,3,32,64),activation='relu') - self.conv3=conv2d(weight_shape=(3,3,64,64),activation='relu') - # Create two dense layers with relu and linear activations - self.dense1=dense([64*4*4,64],activation='relu') - self.dense2=dense([64,10]) - # Store the parameters of the layers in a list - self.param = [self.conv1.weight, - self.conv2.weight, - self.conv3.weight, - self.dense1.weight, - self.dense1.bias, - self.dense2.weight, - self.dense2.bias] - return - - - def data_func(self,data_batch,labels_batch): - # Apply mixup augmentation on the input data and labels - return lmix(data_batch,labels_batch,self.alpha) - - - def fp(self,data): - # Perform forward propagation on the input data - x=tf.nn.conv2d(data,self.conv1.weight,strides=(1,1),padding='SAME') # First convolutional layer - x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # First max pooling layer - x=tf.nn.conv2d(x,self.conv2.weight,strides=(1,1),padding='SAME') # Second convolutional layer - x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # Second max pooling layer - x=tf.nn.conv2d(x,self.conv3.weight,strides=(1,1),padding='SAME') # Third convolutional layer - x=flatten(x) # Flatten the output to a vector - x=tf.nn.relu(tf.matmul(x,self.dense1.weight)+self.dense1.bias) # First dense layer with relu activation - x=tf.nn.dropout(x,rate=0.5) # Apply dropout to prevent overfitting - output=tf.matmul(x,self.dense2.weight)+self.dense2.bias # Output layer with linear activation - output=tf.nn.dropout(output,rate=0.5) # Apply dropout to prevent overfitting - return output - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) - - - def opt(self,gradient): - # Perform optimization on the parameters using the gradient and the batch count - param=self.optimizer.opt(gradient,self.param,self.bc[0]) - return param \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn.py deleted file mode 100644 index 1b16b5c0..00000000 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/nn.py +++ /dev/null @@ -1,39 +0,0 @@ -import tensorflow as tf -import Note.nn.layer.dense as d -from Note.nn.parallel.optimizer import Momentum -from Note.nn.layer.flatten import flatten - -# Define a neural network class -class nn: - def __init__(self): - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.optimizer=Momentum(0.07,0.7) - - - def build(self): - # Create two dense layers with relu and linear activations - self.layer1=d.dense([784,128],activation='relu') - self.layer2=d.dense([128,10]) - # Store the parameters of the layers in a list - self.param=[self.layer1.param,self.layer2.param] - return - - - def fp(self,data): - # Perform forward propagation on the input data - data=flatten(data) - output1=self.layer1.output(data) - output2=self.layer2.output(output1) - return output2 - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) - - - def opt(self,gradient): - # Perform optimization on the parameters using the gradient - param=self.optimizer.opt(gradient,self.param) - return param \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_.py deleted file mode 100644 index d115cc0c..00000000 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_.py +++ /dev/null @@ -1,39 +0,0 @@ -import tensorflow as tf -from Note.nn.parallel.optimizer import Momentum -from Note.nn.layer.flatten import flatten - -# Define a neural network class -class nn: - def __init__(self): - # Initialize the weights and biases of three layers with random values - self.weight1=tf.Variable(tf.random.normal([784,64],dtype='float64')) - self.bias1=tf.Variable(tf.random.normal([64],dtype='float64')) - self.weight2=tf.Variable(tf.random.normal([64,64],dtype='float64')) - self.bias2=tf.Variable(tf.random.normal([64],dtype='float64')) - self.weight3=tf.Variable(tf.random.normal([64,10],dtype='float64')) - self.bias3=tf.Variable(tf.random.normal([10],dtype='float64')) - # Store the parameters of the layers in a list - self.param=[self.weight1,self.weight2,self.weight3,self.bias1,self.bias2,self.bias3] - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.optimizer=Momentum(0.07,0.7) - - - def fp(self,data): - # Perform forward propagation on the input data - data=flatten(data) - layer1=tf.nn.relu(tf.matmul(data,self.weight1)+self.bias1) # First layer with relu activation - layer2=tf.nn.relu(tf.matmul(layer1,self.weight2)+self.bias2) # Second layer with relu activation - output=tf.matmul(layer2,self.weight3)+self.bias3 # Output layer with linear activation - return output - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) - - - def opt(self,gradient): - # Perform optimization on the parameters using the gradient - param=self.optimizer.opt(gradient,self.param) - return param \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_acc.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_acc.py deleted file mode 100644 index 19b055dc..00000000 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_acc.py +++ /dev/null @@ -1,44 +0,0 @@ -import tensorflow as tf -import Note.nn.layer.dense as d -from Note.nn.parallel.optimizer import Momentum -from Note.nn.layer.flatten import flatten -from Note.nn.accuracy import sparse_categorical_accuracy - -# Define a neural network class -class nn: - def __init__(self): - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.optimizer=Momentum(0.07,0.7) - - - def build(self): - # Create two dense layers with relu and linear activations - self.layer1=d.dense([784,128],activation='relu') - self.layer2=d.dense([128,10]) - # Store the parameters of the layers in a list - self.param=[self.layer1.param,self.layer2.param] - return - - - def fp(self,data): - # Perform forward propagation on the input data - data=flatten(data) - output1=self.layer1.output(data) - output2=self.layer2.output(output1) - return output2 - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) - - - def accuracy(self,output,labels): #accuracy function,kernel uses it to calculate accuracy. - return sparse_categorical_accuracy(labels,output) - - - def opt(self,gradient): - # Perform optimization on the parameters using the gradient - param=self.optimizer.opt(gradient,self.param) - return param \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_attenuate.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_attenuate.py deleted file mode 100644 index 2f176151..00000000 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_attenuate.py +++ /dev/null @@ -1,57 +0,0 @@ -import tensorflow as tf -from tensorflow.python.util import nest -import Note.nn.layer.dense as d -from Note.nn.parallel.optimizer import Momentum -from Note.nn.layer.flatten import flatten -from Note.nn.accuracy import sparse_categorical_accuracy - -# Define a neural network class with gradient attenuation -class nn: - def __init__(self): - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.optimizer=Momentum(0.07,0.7) - # Initialize a variable to keep track of the number of optimization steps - self.opt_counter=tf.Variable(tf.zeros(3,dtype=tf.float32)) - - - def build(self): - # Create two dense layers with relu and linear activations - self.layer1=d.dense([784,128],activation='relu') - self.layer2=d.dense([128,10]) - # Store the parameters of the layers in a list - self.param=[self.layer1.param,self.layer2.param] - return - - - def fp(self,data): - # Perform forward propagation on the input data - data=flatten(data) # Flatten the data to a one-dimensional vector - output1=self.layer1.output(data) # Apply the first layer to the data and get the output - output2=self.layer2.output(output1) # Apply the second layer to the output of the first layer and get the final output - return output2 - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) # Use the loss function to calculate the loss - - - def accuracy(self,output,labels): #accuracy function,kernel uses it to calculate accuracy. - return sparse_categorical_accuracy(labels,output) - - - def attenuate(self,gradient,oc,p): #gradient attenuation function,kernel uses it to calculate attenuation coefficient. - # Apply an exponential decay to the gradient based on the optimization counter - ac=0.9**oc[0][p] #ac:attenuation coefficient - gradient_flat=nest.flatten(gradient) # Flatten the gradient to a one-dimensional vector - for i in range(len(gradient_flat)): #oc:optimization counter - gradient_flat[i]=ac*gradient_flat[i] #p:process number - gradient=nest.pack_sequence_as(gradient,gradient_flat) # Restore the gradient to its original shape - return gradient - - - def opt(self,gradient): - # Perform optimization on the parameters using the gradient - param=self.optimizer.opt(gradient,self.param) # Use the optimizer to update the parameters - return param \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_clipping.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_clipping.py deleted file mode 100644 index a0f757d3..00000000 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_clipping.py +++ /dev/null @@ -1,47 +0,0 @@ -import tensorflow as tf -import Note.nn.layer.dense as d -from Note.nn.parallel.optimizer import Momentum -from Note.nn.layer.flatten import flatten -from Note.nn.gradient_clipping import gradient_clipping - -# Define a neural network class -class nn: - def __init__(self): - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.optimizer=Momentum(0.07,0.7) - - - def build(self): - # Create two dense layers with relu and linear activations - self.layer1=d.dense([784,128],activation='relu') - self.layer2=d.dense([128,10]) - # Store the parameters of the layers in a list - self.param=[self.layer1.param,self.layer2.param] - return - - - def fp(self,data): - # Perform forward propagation on the input data - data=flatten(data) - output1=self.layer1.output(data) - output2=self.layer2.output(output1) - return output2 - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) - - - def gradient(self,tape,loss): - # Compute the gradients of the loss with respect to the parameters - grads=tape.gradient(loss,self.param) - # Clip the gradients by value to avoid exploding gradients - return gradient_clipping(grads,'value',1.0) - - - def opt(self,gradient): - # Perform optimization on the parameters using the gradient - param=self.optimizer.opt(gradient,self.param) - return param \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_device.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_device.py deleted file mode 100644 index e48acac3..00000000 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_device.py +++ /dev/null @@ -1,40 +0,0 @@ -import tensorflow as tf # import TensorFlow library -import Note.nn.layer.dense as d # import Note's dense layer module -from Note.nn.layer.flatten import flatten # import Note's flatten layer function -from Note.nn.parallel.optimizer import Momentum # import Note's momentum optimizer module -from Note.nn.parallel.assign_device import assign_device # import the function to assign device according to the process index and the device type - - -class nn: # A neural network class example, allocate device for multiple threads - def __init__(self): # initialize the network - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) # loss object, kernel uses it to calculate loss. Here we use sparse categorical crossentropy with logits as output - self.optimizer=Momentum(0.07,0.7) # optimizer, kernel uses it to optimize. Here we use a custom momentum optimizer with learning rate 0.07 and momentum 0.7 - - - def build(self): # build function, kernel uses it to create the network layers - # Create two dense layers with relu and linear activations - self.layer1=d.dense([784,128],activation='relu') # the first layer with 784 input units and 128 output units and ReLU activation - self.layer2=d.dense([128,10]) # the second layer with 128 input units and 10 output units and linear activation - # Store the parameters of the layers in a list - self.param=[self.layer1.param,self.layer2.param] # parameter list of both layers, kernel uses it list for backpropagation - return - - - def fp(self,data,p): # forward propagation function, kernel uses it for forward propagation - with tf.device(assign_device(p,'GPU')): # assign the device according to the process index p - data=flatten(data) # flatten the data to a vector of 784 elements - output1=self.layer1.output(data) # pass the data through the first layer and get the output - output2=self.layer2.output(output1) # pass the output of the first layer through the second layer and get the final output logits - return output2 - - - def loss(self,output,labels,p): # loss function, kernel uses it to calculate loss - with tf.device(assign_device(p,'GPU')): # assign the device according to the process index p - return self.loss_object(labels,output) # return the mean softmax cross entropy loss between labels and output - - - def opt(self,gradient,p): # optimization function, kernel uses it to optimize parameter - # Perform optimization on the parameters using the gradient - with tf.device(assign_device(p,'GPU')): # assign the device according to the process index p - param=self.optimizer.opt(gradient,self.param) # apply the Note's momentum optimizer to update the parameters using the gradient - return param \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/self_LSTM.py b/7.0/neuralnetwork/DL/tensorflow/parallel/self_LSTM.py deleted file mode 100644 index 85af312a..00000000 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/self_LSTM.py +++ /dev/null @@ -1,54 +0,0 @@ -import tensorflow as tf -from Note.nn.parallel.optimizer import Adam -from Note.nn.layer.LSTM import LSTM -from Note.nn.layer.self_attention import self_attention -from Note.nn.layer.dense import dense -from Note.nn.layer.emdedding import embedding - -# Define a LSTM self attention neural network class -class self_LSTM: - def __init__(self,vocab_size,embed_size,num_heads,num_layers): - self.vocab_size=vocab_size - self.embed_size=embed_size - self.num_heads=num_heads - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.optimizer=Adam() - # Initialize a variable to keep track of the batch count - self.bc=tf.Variable(0,dtype=tf.float32) - - - def build(self): - # Create an embedding layer to map input tokens to vectors - self.embedding_layer = embedding(self.vocab_size,self.embed_size) - # Create a LSTM layer to process the input sequence - self.lstm_layer = LSTM(weight_shape=[self.embed_size,self.embed_size],return_sequence=True) - # Create a self attention layer to attend to the LSTM output - self.self_attention_layer = self_attention(weight_shape=[self.embed_size,self.embed_size],num_heads=self.num_heads) - # Create a linear layer to map the attention output to logits - self.logits_layer = dense([self.embed_size,self.vocab_size]) - # Store the parameters of the layers in a list - self.param = [self.embedding_layer.embeddings] - self.param.extend(self.lstm_layer.param) - self.param.extend(self.self_attention_layer.param) - self.param.extend(self.logits_layer.param) - - - def fp(self,data): - # Perform forward propagation on the input data - x = self.embedding_layer.output(data) # shape: (batch_size, seq_len, embed_size) - x = self.lstm_layer.output(x) # shape: (batch_size, seq_len, embed_size) - x, attention_weights = self.self_attention_layer.output(x,self.num_heads) # shape: (batch_size, seq_len, embed_size) - output = self.logits_layer.output(x) # shape: (batch_size, seq_len, vocab_size) - return output - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) - - - def opt(self,gradient): - # Perform optimization on the parameters using the gradient and the batch count - param=self.optimizer.opt(gradient,self.param,self.bc[0]) - return param \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/transformer.py b/7.0/neuralnetwork/DL/tensorflow/parallel/transformer.py deleted file mode 100644 index ae052538..00000000 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/transformer.py +++ /dev/null @@ -1,55 +0,0 @@ -import tensorflow as tf -from Note.nn.parallel.optimizer import Adam -from Note.nn.layer.transformer import transformer -from Note.nn.layer.dense import dense -from Note.nn.layer.emdedding import embedding -from Note.nn.positional_encoding import positional_encoding - -# Define a positional encoding transformer neural network class -class Transformer: - def __init__(self,vocab_size,embed_size,num_heads,num_layers,max_len): - self.vocab_size=vocab_size - self.embed_size=embed_size - self.num_heads=num_heads - self.num_layers=num_layers - self.max_len=max_len - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.optimizer=Adam() - # Initialize a variable to keep track of the batch count - self.bc=tf.Variable(0,dtype=tf.float32) - - - def build(self): - # Create an embedding layer to map input tokens to vectors - self.embedding_layer = embedding(self.vocab_size,self.embed_size) - # Create a list of transformer layers - self.transformer_layers = [transformer(weight_shape=[self.embed_size, self.embed_size], num_heads=self.num_heads) for _ in range(self.num_layers)] - # Create a linear layer to map the transformer output to logits - self.logits_layer = dense([self.embed_size,self.vocab_size]) - # Store the parameters of the layers in a list - self.param = [self.embedding_layer.embeddings] - for transformer_layer in self.transformer_layers: - self.param.extend(transformer_layer.param) - self.param.extend(self.logits_layer.param) - - - def fp(self,data): - # Perform forward propagation on the input data - x = self.embedding_layer.output(data) # shape: (batch_size, seq_len, embed_size) - x += positional_encoding(self.max_len,self.embed_size) # shape: (batch_size, seq_len, embed_size) - for transformer_layer in self.transformer_layers: - x = transformer_layer.output(x) # shape: (batch_size, seq_len, embed_size) - output = self.logits_layer.output(x) # shape: (batch_size, seq_len, vocab_size) - return output - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) - - - def opt(self,gradient): - # Perform optimization on the parameters using the gradient and the batch count - param=self.optimizer.opt(gradient,self.param,self.bc[0]) - return param \ No newline at end of file diff --git a/7.0/neuralnetwork/RL/pytorch/non-parallel/DDPG.py b/7.0/neuralnetwork/RL/pytorch/non-parallel/DDPG.py deleted file mode 100644 index 4b0abd94..00000000 --- a/7.0/neuralnetwork/RL/pytorch/non-parallel/DDPG.py +++ /dev/null @@ -1,101 +0,0 @@ -import torch -import torch.nn.functional as F -import numpy as np -import gym - - -class actor(torch.nn.Module): - def __init__(self,state_dim,hidden_dim,action_dim,action_bound): - super(actor,self).__init__() - self.fc1=torch.nn.Linear(state_dim,hidden_dim) - self.fc2=torch.nn.Linear(hidden_dim,action_dim) - self.action_bound=action_bound - - - def forward(self,x): - x=F.relu(self.fc1(x)) - return torch.tanh(self.fc2(x))*self.action_bound - - -class critic(torch.nn.Module): - def __init__(self,state_dim,hidden_dim,action_dim): - super(critic,self).__init__() - self.fc1=torch.nn.Linear(state_dim+action_dim,hidden_dim) - self.fc2=torch.nn.Linear(hidden_dim,1) - - - def forward(self,x,a): - cat=torch.cat([x,a],dim=1) - x=F.relu(self.fc1(cat)) - return self.fc2(x) - - -class DDPG: - def __init__(self,hidden_dim,sigma,gamma,tau,actor_lr,critic_lr): - if torch.cuda.is_available(): - self.device=torch.device('cuda') - else: - self.device=torch.device('cpu') - self.genv=gym.make('Pendulum-v0') - state_dim=self.genv.observation_space.shape[0] - action_dim=self.genv.action_space.shape[0] - action_bound=self.genv.action_space.high[0] - self.actor=actor(state_dim,hidden_dim,action_dim,action_bound).to(self.device) - self.critic=critic(state_dim,hidden_dim,action_dim).to(self.device) - self.target_actor=actor(state_dim,hidden_dim,action_dim,action_bound).to(self.device) - self.target_critic=critic(state_dim,hidden_dim,action_dim).to(self.device) - self.target_actor.load_state_dict(self.actor.state_dict()) - self.target_critic.load_state_dict(self.critic.state_dict()) - self.sigma=sigma - self.gamma=gamma - self.tau=tau - self.actor_opt=torch.optim.Adam(self.actor.parameters(),lr=actor_lr) - self.critic_opt=torch.optim.Adam(self.critic.parameters(),lr=critic_lr) - - - def noise(self): - return np.random.normal(scale=self.sigma) - - - def env(self,a=None,initial=None): - if initial==True: - state=self.genv.reset(seed=0) - return state - else: - next_state,reward,done,_=self.genv.step(a) - return next_state,reward,done - - - def loss(self,s,a,next_s,r,d): - s=torch.tensor(s,dtype=torch.float).to(self.device) - a=torch.tensor(a,dtype=torch.float).view(-1,1).to(self.device) - next_s=torch.tensor(next_s,dtype=torch.float).to(self.device) - r=torch.tensor(r,dtype=torch.float).view(-1,1).to(self.device) - d=torch.tensor(d,dtype=torch.float).view(-1,1).to(self.device) - next_q_value=self.target_critic(next_s,self.target_actor(next_s)) - q_target=r+self.gamma*next_q_value*(1-d) - actor_loss=-torch.mean(self.critic(s,self.actor(s))) - critic_loss=F.mse_loss(self.critic(s,a),q_target) - return [actor_loss,critic_loss] - - - def backward(self,loss): - self.actor_opt.zero_grad() - loss[0].backward() - self.critic_opt.zero_grad() - loss[1].backward() - return - - - def opt(self): - self.actor_opt.step() - self.critic_opt.step() - return - - - def update_param(self): - for target_param,param in zip(self.target_actor.parameters(),self.actor.parameters()): - target_param.data.copy_(target_param.data*(1.0-self.tau)+param.data*self.tau) - for target_param,param in zip(self.target_critic.parameters(),self.critic.parameters()): - target_param.data.copy_(target_param.data*(1.0-self.tau)+param.data*self.tau) - return \ No newline at end of file diff --git a/7.0/neuralnetwork/RL/pytorch/non-parallel/DQN.py b/7.0/neuralnetwork/RL/pytorch/non-parallel/DQN.py deleted file mode 100644 index a0e58689..00000000 --- a/7.0/neuralnetwork/RL/pytorch/non-parallel/DQN.py +++ /dev/null @@ -1,64 +0,0 @@ -import torch -import gym -import torch.nn.functional as F - - -class Qnet(torch.nn.Module): - def __init__(self,state_dim,hidden_dim,action_dim): - super(Qnet,self).__init__() - self.fc1=torch.nn.Linear(state_dim,hidden_dim) - self.fc2=torch.nn.Linear(hidden_dim,action_dim) - - - def forward(self,x): - x=F.relu(self.fc1(x)) - return self.fc2(x) - - -class DQN: - def __init__(self,state_dim,hidden_dim,action_dim): - if torch.cuda.is_available(): - self.device=torch.device('cuda') - else: - self.device=torch.device('cpu') - self.nn=Qnet(state_dim,hidden_dim,action_dim).to(self.device) - self.target_q_net=Qnet(state_dim,hidden_dim,action_dim).to(self.device) - self.optimizer=torch.optim.Adam(self.nn.parameters(),lr=2e-3) #optimizer,kernel uses it to optimize. - self.genv=gym.make('CartPole-v0') #create environment - - - def env(self,a=None,initial=None): #environment function,kernel uses it to interact with the environment. - if initial==True: - state=self.genv.reset() - return state - else: - next_state,reward,done,_=self.genv.step(a) - return next_state,reward,done - - - def loss(self,s,a,next_s,r,d): #loss function,kernel uses it to calculate loss. - s=torch.tensor(s,dtype=torch.float).to(self.device) - a=torch.tensor(a,dtype=torch.int64).view(-1,1).to(self.device) - next_s=torch.tensor(next_s,dtype=torch.float).to(self.device) - r=torch.tensor(r,dtype=torch.float).view(-1,1).to(self.device) - d=torch.tensor(d,dtype=torch.float).view(-1,1).to(self.device) - q_value=self.nn(s).gather(1,a) - next_q_value=self.target_q_net(next_s).max(1)[0].view(-1,1) - target=r+0.98*next_q_value*(1-d) - return F.mse_loss(q_value,target) - - - def backward(self,loss): #backward function,kernel uses it for backpropagation. - self.optimizer.zero_grad() - loss.backward() - return - - - def opt(self): #opt function,kernel uses it to optimize. - self.optimizer.step() - return - - - def update_param(self): #update function,kernel uses it to update parameter. - self.target_q_net.load_state_dict(self.nn.state_dict()) - return \ No newline at end of file diff --git a/7.0/neuralnetwork/RL/pytorch/non-parallel/DQN_pr.py b/7.0/neuralnetwork/RL/pytorch/non-parallel/DQN_pr.py deleted file mode 100644 index 66c71847..00000000 --- a/7.0/neuralnetwork/RL/pytorch/non-parallel/DQN_pr.py +++ /dev/null @@ -1,76 +0,0 @@ -import torch -import gym -import torch.nn.functional as F -import Note.create.RL.rl.prioritized_replay as pr - -#prioritized replay example -class Qnet(torch.nn.Module): - def __init__(self,state_dim,hidden_dim,action_dim): - super(Qnet,self).__init__() - self.fc1=torch.nn.Linear(state_dim,hidden_dim) - self.fc2=torch.nn.Linear(hidden_dim,action_dim) - - - def forward(self,x): - x=F.relu(self.fc1(x)) - return self.fc2(x) - - -class DQN: - def __init__(self,state_dim,hidden_dim,action_dim): - if torch.cuda.is_available(): - self.device=torch.device('cuda') - else: - self.device=torch.device('cpu') - self.nn=Qnet(state_dim,hidden_dim,action_dim).to(self.device) - self.target_q_net=Qnet(state_dim,hidden_dim,action_dim).to(self.device) - self.pr=pr.pr() - self.initial_TD=7 - self._epsilon=0.0007 - self.alpha=0.7 - self.optimizer=torch.optim.Adam(self.nn.parameters(),lr=2e-3) - self.genv=gym.make('CartPole-v0') - - - def env(self,a=None,initial=None): - if initial==True: - state=self.genv.reset(seed=0) - return state - else: - next_state,reward,done,_=self.genv.step(a) - return next_state,reward,done - - - def data_func(self,state_pool,action_pool,next_state_pool,reward_pool,done_pool,batch): - s,a,next_s,r,d=self.pr.sample(state_pool,action_pool,next_state_pool,reward_pool,done_pool,self.epsilon,self.alpha,batch) - return s,a,next_s,r,d - - - def loss(self,s,a,next_s,r,d): - s=torch.tensor(s,dtype=torch.float).to(self.device) - a=torch.tensor(a,dtype=torch.int64).view(-1,1).to(self.device) - next_s=torch.tensor(next_s,dtype=torch.float).to(self.device) - r=torch.tensor(r,dtype=torch.float).view(-1,1).to(self.device) - d=torch.tensor(d,dtype=torch.float).view(-1,1).to(self.device) - q_value=self.nn(s).gather(1,a) - next_q_value=self.target_q_net(next_s).max(1)[0].view(-1,1) - target=r+0.98*next_q_value*(1-d) - TD=target-q_value - self.pr.update_TD(TD) - return torch.mean(TD**2) - - - def backward(self,loss): - self.optimizer.zero_grad() - loss.backward() - return - - - def opt(self): - self.optimizer.step() - return - - - def update_param(self): - self.target_q_net.load_state_dict(self.nn.state_dict()) - return \ No newline at end of file diff --git a/7.0/neuralnetwork/RL/pytorch/non-parallel/DoubleDQN.py b/7.0/neuralnetwork/RL/pytorch/non-parallel/DoubleDQN.py deleted file mode 100644 index 31e606e8..00000000 --- a/7.0/neuralnetwork/RL/pytorch/non-parallel/DoubleDQN.py +++ /dev/null @@ -1,65 +0,0 @@ -import torch -import gym -import torch.nn.functional as F - - -class Qnet(torch.nn.Module): - def __init__(self,state_dim,hidden_dim,action_dim): - super(Qnet,self).__init__() - self.fc1=torch.nn.Linear(state_dim,hidden_dim) - self.fc2=torch.nn.Linear(hidden_dim,action_dim) - - - def forward(self,x): - x=F.relu(self.fc1(x)) - return self.fc2(x) - - -class DoubleDQN: - def __init__(self,state_dim,hidden_dim,action_dim): - if torch.cuda.is_available(): - self.device=torch.device('cuda') - else: - self.device=torch.device('cpu') - self.nn=Qnet(state_dim,hidden_dim,action_dim).to(self.device) - self.target_q_net=Qnet(state_dim,hidden_dim,action_dim).to(self.device) - self.optimizer=torch.optim.Adam(self.nn.parameters(),lr=2e-3) - self.genv=gym.make('CartPole-v0') - - - def env(self,a=None,initial=None): - if initial==True: - state=self.genv.reset(seed=0) - return state - else: - next_state,reward,done,_=self.genv.step(a) - return next_state,reward,done - - - def loss(self,s,a,next_s,r,d): - s=torch.tensor(s,dtype=torch.float).to(self.device) - a=torch.tensor(a,dtype=torch.int64).view(-1,1).to(self.device) - next_s=torch.tensor(next_s,dtype=torch.float).to(self.device) - r=torch.tensor(r,dtype=torch.float).view(-1,1).to(self.device) - d=torch.tensor(d,dtype=torch.float).view(-1,1).to(self.device) - q_value=self.nn(s).gather(1,a) - max_action=self.nn(next_s).max(1)[1].view(-1,1) - next_q_value=self.target_q_net(next_s).gather(1,max_action) - target=r+0.98*next_q_value*(1-d) - return F.mse_loss(q_value,target) - - - def backward(self,loss): - self.optimizer.zero_grad() - loss.backward() - return - - - def opt(self): - self.optimizer.step() - return - - - def update_param(self): - self.target_q_net.load_state_dict(self.nn.state_dict()) - return \ No newline at end of file diff --git a/7.0/neuralnetwork/RL/pytorch/non-parallel/DuelingDQN.py b/7.0/neuralnetwork/RL/pytorch/non-parallel/DuelingDQN.py deleted file mode 100644 index b81eeb37..00000000 --- a/7.0/neuralnetwork/RL/pytorch/non-parallel/DuelingDQN.py +++ /dev/null @@ -1,67 +0,0 @@ -import torch -import gym -import torch.nn.functional as F - - -class VAnet(torch.nn.Module): - def __init__(self,state_dim,hidden_dim,action_dim): - super(VAnet,self).__init__() - self.fc1=torch.nn.Linear(state_dim,hidden_dim) - self.fc_A=torch.nn.Linear(hidden_dim,action_dim) - self.fc_V=torch.nn.Linear(hidden_dim,1) - - - def forward(self,x): - A=self.fc_A(F.relu(self.fc1(x))) - V=self.fc_V(F.relu(self.fc1(x))) - Q=V+A-A.mean(1).view(-1,1) - return Q - - -class DuelingDQN: - def __init__(self,state_dim,hidden_dim,action_dim): - if torch.cuda.is_available(): - self.device=torch.device('cuda') - else: - self.device=torch.device('cpu') - self.nn=VAnet(state_dim,hidden_dim,action_dim).to(self.device) - self.target_q_net=VAnet(state_dim,hidden_dim,action_dim).to(self.device) - self.optimizer=torch.optim.Adam(self.nn.parameters(),lr=2e-3) - self.genv=gym.make('CartPole-v0') - - - def env(self,a=None,initial=None): - if initial==True: - state=self.genv.reset(seed=0) - return state - else: - next_state,reward,done,_=self.genv.step(a) - return next_state,reward,done - - - def loss(self,s,a,next_s,r,d): - s=torch.tensor(s,dtype=torch.float).to(self.device) - a=torch.tensor(a,dtype=torch.int64).view(-1,1).to(self.device) - next_s=torch.tensor(next_s,dtype=torch.float).to(self.device) - r=torch.tensor(r,dtype=torch.float).view(-1,1).to(self.device) - d=torch.tensor(d,dtype=torch.float).view(-1,1).to(self.device) - q_value=self.nn(s).gather(1,a) - next_q_value=self.target_q_net(next_s).max(1)[0].view(-1,1) - target=r+0.98*next_q_value*(1-d) - return F.mse_loss(q_value,target) - - - def backward(self,loss): - self.optimizer.zero_grad() - loss.backward() - return - - - def opt(self): - self.optimizer.step() - return - - - def update_param(self): - self.target_q_net.load_state_dict(self.nn.state_dict()) - return \ No newline at end of file diff --git a/7.0/neuralnetwork/RL/pytorch/parallel/DQN.py b/7.0/neuralnetwork/RL/pytorch/parallel/DQN.py deleted file mode 100644 index f4d8417d..00000000 --- a/7.0/neuralnetwork/RL/pytorch/parallel/DQN.py +++ /dev/null @@ -1,65 +0,0 @@ -import torch -import gym -import torch.nn.functional as F -from Note.nn.parallel.assign_device_pytorch import assign_device - - -class Qnet(torch.nn.Module): - def __init__(self,state_dim,hidden_dim,action_dim): - super(Qnet,self).__init__() - self.fc1=torch.nn.Linear(state_dim,hidden_dim) - self.fc2=torch.nn.Linear(hidden_dim,action_dim) - - - def forward(self,x): - x=F.relu(self.fc1(x)) - return self.fc2(x) - - -class DQN: - def __init__(self,state_dim,hidden_dim,action_dim): - if torch.cuda.is_available(): - self.device=torch.device('cuda') - else: - self.device=torch.device('cpu') - self.nn=Qnet(state_dim,hidden_dim,action_dim).to(self.device) - self.target_q_net=Qnet(state_dim,hidden_dim,action_dim).to(self.device) - self.optimizer=[torch.optim.Adam(self.nn.parameters(),lr=2e-3) for _ in range(5)] #optimizer,kernel uses it to optimize. - self.genv=[gym.make('CartPole-v0') for _ in range(5)] #create environment - - - def env(self,a=None,p=None,initial=None): #environment function,kernel uses it to interact with the environment. - if initial==True: - state=self.genv[p].reset() - return state - else: - next_state,reward,done,_=self.genv[p].step(a) - return next_state,reward,done - - - def loss(self,s,a,next_s,r,d,p): #loss function,kernel uses it to calculate loss. - s=torch.tensor(s,dtype=torch.float).to(assign_device(p,'GPU')) - a=torch.tensor(a,dtype=torch.int64).view(-1,1).to(assign_device(p,'GPU')) - next_s=torch.tensor(next_s,dtype=torch.float).to(assign_device(p,'GPU')) - r=torch.tensor(r,dtype=torch.float).view(-1,1).to(assign_device(p,'GPU')) - d=torch.tensor(d,dtype=torch.float).view(-1,1).to(assign_device(p,'GPU')) - q_value=self.nn(s).gather(1,a) - next_q_value=self.target_q_net(next_s).max(1)[0].view(-1,1) - target=r+0.98*next_q_value*(1-d) - return F.mse_loss(q_value,target) - - - def backward(self,loss,p): #backward function,kernel uses it for backpropagation. - self.optimizer[p].zero_grad(set_to_none=True) - loss.backward() - return - - - def opt(self,p): #opt function,kernel uses it to optimize. - self.optimizer[p].step() - return - - - def update_param(self): #update function,kernel uses it to update parameter. - self.target_q_net.load_state_dict(self.nn.state_dict()) - return \ No newline at end of file diff --git a/7.0/neuralnetwork/RL/tensorflow/non-parallel/DDPG.py b/7.0/neuralnetwork/RL/tensorflow/non-parallel/DDPG.py deleted file mode 100644 index d3666ee1..00000000 --- a/7.0/neuralnetwork/RL/tensorflow/non-parallel/DDPG.py +++ /dev/null @@ -1,84 +0,0 @@ -import tensorflow as tf # import TensorFlow library -import numpy as np # import NumPy library -import gym # import OpenAI Gym library - - -class actor: # define a class for the actor network - def __init__(self,state_dim,hidden_dim,action_dim,action_bound): # initialize the network with state dimension, hidden dimension, action dimension and action bound - self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) # create a weight matrix for the first layer - self.bias1=tf.Variable(tf.random.normal([hidden_dim])) # create a bias vector for the first layer - self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) # create a weight matrix for the second layer - self.bias2=tf.Variable(tf.random.normal([action_dim])) # create a bias vector for the second layer - self.action_bound=action_bound # store the action bound - self.param=[self.weight1,self.bias1,self.weight2,self.bias2] # store the network parameters in a list - - - def fp(self,x): # forward propagation function, kernel uses it for forward propagation - x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) # apply the first layer with ReLU activation - return tf.nn.tanh(tf.matmul(x,self.weight2)+self.bias2)*self.action_bound # apply the second layer with tanh activation and scale it by the action bound - - -class critic: # define a class for the critic network - def __init__(self,state_dim,hidden_dim,action_dim): # initialize the network with state dimension, hidden dimension and action dimension - self.weight1=tf.Variable(tf.random.normal([state_dim+action_dim,hidden_dim])) # create a weight matrix for the first layer - self.bias1=tf.Variable(tf.random.normal([hidden_dim])) # create a bias vector for the first layer - self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) # create a weight matrix for the second layer - self.bias2=tf.Variable(tf.random.normal([action_dim])) # create a bias vector for the second layer - self.param=[self.weight1,self.bias1,self.weight2,self.bias2] # store the network parameters in a list - - - def fp(self,x,a): # forward propagation function, kernel uses it for forward propagation - cat=tf.concat([x,a],axis=1) # concatenate the state and action vectors along the first axis - x=tf.nn.relu(tf.matmul(cat,self.weight1)+self.bias1) # apply the first layer with ReLU activation - return tf.matmul(x,self.weight2)+self.bias2 # apply the second layer and return the output - - -class DDPG: # define a class for the DDPG agent - def __init__(self,hidden_dim,sigma,gamma,tau,actor_lr,critic_lr): # initialize the agent with hidden dimension, noise scale, discount factor, soft update factor, actor learning rate and critic learning rate - self.genv=gym.make('Pendulum-v0') # create environment - state_dim=self.genv.observation_space.shape[0] # get the state dimension from the environment observation space - action_dim=self.genv.action_space.shape[0] # get the action dimension from the environment action space - action_bound=self.genv.action_space.high[0] # get the action bound from the environment action space high value (assume symmetric) - self.actor=actor(state_dim,hidden_dim,action_dim,action_bound) # create an actor network for the agent - self.critic=critic(state_dim,hidden_dim,action_dim) # create a critic network for the agent - self.target_actor=actor(state_dim,hidden_dim,action_dim,action_bound) # create a target actor network for the agent - self.target_critic=critic(state_dim,hidden_dim,action_dim) # create a target critic network for the agent - self.actor_param=self.actor.param # parameter list of actor network, kernel uses it list for backpropagation - self.critic_param=self.critic.param # parameter list of critic network, kernel uses it list for backpropagation - self.target_actor.param=self.actor_param.copy() # copy the parameters from actor network to target actor network - self.target_critic.param=self.critic_param.copy() # copy the parameters from critic network to target critic network - self.param=[self.actor_param,self.critic_param] # parameter list of both networks, kernel uses it list for backpropagation - self.sigma=sigma # noise scale - self.gamma=gamma # discount factor - self.tau=tau # soft update factor - self.opt=tf.keras.optimizers.Adam() # optimizer, kernel uses it to optimize. Here we use Adam optimizer - - - def noise(self): # noise function, kernel uses it to generate exploration noise - return np.random.normal(scale=self.sigma) # return a random sample from a normal distribution with zero mean and sigma scale - - - def env(self,a=None,initial=None): # environment function, kernel uses it to interact with the environment - if initial==True: # if initial is True, reset the environment and return the initial state - state=self.genv.reset(seed=0) - return state - else: # otherwise, take an action and return the next state, reward and done flag - next_state,reward,done,_=self.genv.step(a) - return next_state,reward,done - - - def loss(self,s,a,next_s,r,d): # loss function, kernel uses it to calculate loss - a=tf.expand_dims(a,axis=1) # expand the action vector to match the Q-value matrix shape - next_q_value=self.target_critic.fp(next_s,self.target_actor.fp(next_s)) # get the Q-value for the next state and action from the target critic network - q_target=r+self.gamma*next_q_value*(1-d) # calculate the target value using Bellman equation with discount factor gamma - actor_loss=-tf.reduce_mean(self.critic.fp(s,self.actor.fp(s))) # calculate the actor loss as the negative mean of the Q-value for the current state and action from the critic network - critic_loss=tf.reduce_mean((self.critic.fp(s,a)-q_target)**2) # calculate the critic loss as the mean squared error between Q-value and target value - return [actor_loss,critic_loss] # return a list of actor loss and critic loss - - - def update_param(self): # update function, kernel uses it to update parameter - for target_param,param in zip(self.target_actor.param,self.actor.param): # for each pair of parameters in target actor network and actor network - target_param=target_param*(1.0-self.tau)+param*self.tau # update the target parameter using soft update with factor tau - for target_param,param in zip(self.target_critic.param,self.critic.param): # for each pair of parameters in target critic network and critic network - target_param=target_param*(1.0-self.tau)+param*self.tau # update the target parameter using soft update with factor tau - return \ No newline at end of file diff --git a/7.0/neuralnetwork/RL/tensorflow/non-parallel/DQN.py b/7.0/neuralnetwork/RL/tensorflow/non-parallel/DQN.py deleted file mode 100644 index 21487674..00000000 --- a/7.0/neuralnetwork/RL/tensorflow/non-parallel/DQN.py +++ /dev/null @@ -1,46 +0,0 @@ -import tensorflow as tf # import TensorFlow library -import gym # import OpenAI Gym library - - -class Qnet: # define a class for the Q-network - def __init__(self,state_dim,hidden_dim,action_dim): # initialize the network with state dimension, hidden dimension and action dimension - self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) # create a weight matrix for the first layer - self.bias1=tf.Variable(tf.random.normal([hidden_dim])) # create a bias vector for the first layer - self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) # create a weight matrix for the second layer - self.bias2=tf.Variable(tf.random.normal([action_dim])) # create a bias vector for the second layer - self.param=[self.weight1,self.bias1,self.weight2,self.bias2] # store the network parameters in a list - - def fp(self,x): # forward propagation function, kernel uses it for forward propagation - x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) # apply the first layer with ReLU activation - return tf.matmul(x,self.weight2)+self.bias2 # apply the second layer and return the output - - -class DQN: # define a class for the DQN agent - def __init__(self,state_dim,hidden_dim,action_dim): # initialize the agent with state dimension, hidden dimension and action dimension - self.nn=Qnet(state_dim,hidden_dim,action_dim) # create a Q-network for the agent - self.target_q_net=Qnet(state_dim,hidden_dim,action_dim) # create a target Q-network for the agent - self.param=self.nn.param # parameter list, kernel uses it list for backpropagation - self.opt=tf.keras.optimizers.Adam() # optimizer, kernel uses it to optimize - self.genv=gym.make('CartPole-v0') # create environment - - - def env(self,a=None,initial=None): # environment function, kernel uses it to interact with the environment - if initial==True: # if initial is True, reset the environment and return the initial state - state=self.genv.reset(seed=0) - return state - else: # otherwise, take an action and return the next state, reward and done flag - next_state,reward,done,_=self.genv.step(a) - return next_state,reward,done - - - def loss(self,s,a,next_s,r,d): # loss function, kernel uses it to calculate loss - a=tf.expand_dims(a,axis=1) # expand the action vector to match the Q-value matrix shape - q_value=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) # get the Q-value for the selected action - next_q_value=tf.reduce_max(self.target_q_net.fp(next_s),axis=1) # get the maximum Q-value for the next state from the target network - target=r+0.98*next_q_value*(1-d) # calculate the target value using Bellman equation with discount factor 0.98 - return tf.reduce_mean((q_value-target)**2) # return the mean squared error between Q-value and target value - - - def update_param(self): # update function, kernel uses it to update parameter - self.target_q_net.param=self.param.copy() # copy the parameters from the Q-network to the target network - return \ No newline at end of file diff --git a/7.0/neuralnetwork/RL/tensorflow/non-parallel/DQN_pr.py b/7.0/neuralnetwork/RL/tensorflow/non-parallel/DQN_pr.py deleted file mode 100644 index 77e8b8a9..00000000 --- a/7.0/neuralnetwork/RL/tensorflow/non-parallel/DQN_pr.py +++ /dev/null @@ -1,59 +0,0 @@ -import tensorflow as tf -import gym -import Note.create.RL.rl.prioritized_replay as pr - - -class Qnet: - def __init__(self,state_dim,hidden_dim,action_dim): - self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) - self.bias1=tf.Variable(tf.random.normal([hidden_dim])) - self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) - self.bias2=tf.Variable(tf.random.normal([action_dim])) - self.param=[self.weight1,self.bias1,self.weight2,self.bias2] - - def fp(self,x): - x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) - return tf.matmul(x,self.weight2)+self.bias2 - - -class DQN: - def __init__(self,state_dim,hidden_dim,action_dim): - self.nn=Qnet(state_dim,hidden_dim,action_dim) - self.target_q_net=Qnet(state_dim,hidden_dim,action_dim) - self.param=self.nn.param - self.pr=pr.pr() - self.initial_TD=7 - self._epsilon=0.0007 - self.alpha=0.7 - self.opt=tf.keras.optimizers.Adam() - self.genv=gym.make('CartPole-v0') - - - def env(self,a=None,initial=None): - if initial==True: - state=self.genv.reset(seed=0) - return state - else: - next_state,reward,done,_=self.genv.step(a) - return next_state,reward,done - - - def data_func(self,state_pool,action_pool,next_state_pool,reward_pool,done_pool,batch): - s,a,next_s,r,d=self.pr.sample(state_pool,action_pool,next_state_pool,reward_pool,done_pool,self.epsilon,self.alpha,batch) - return s,a,next_s,r,d - - - def loss(self,s,a,next_s,r,d): - a=tf.expand_dims(a,axis=1) - q_value=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) - next_q_value=tf.reduce_max(self.target_q_net.fp(next_s),axis=1) - target=r+0.98*next_q_value*(1-d) - target=tf.expand_dims(target,axis=1) - TD=target-q_value - self.pr.update_TD(TD) - return tf.reduce_mean(TD**2) - - - def update_param(self): - self.target_q_net.param=self.param.copy() - return \ No newline at end of file diff --git a/7.0/neuralnetwork/RL/tensorflow/parallel/DQN.py b/7.0/neuralnetwork/RL/tensorflow/parallel/DQN.py deleted file mode 100644 index d7756475..00000000 --- a/7.0/neuralnetwork/RL/tensorflow/parallel/DQN.py +++ /dev/null @@ -1,52 +0,0 @@ -import tensorflow as tf # import TensorFlow library -import gym # import OpenAI Gym library -import Note.nn.parallel.optimizer as o # import Note's optimizer module - - -class Qnet: # define a class for the Q-network - def __init__(self,state_dim,hidden_dim,action_dim): # initialize the network with state dimension, hidden dimension and action dimension - self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) # create a weight matrix for the first layer - self.bias1=tf.Variable(tf.random.normal([hidden_dim])) # create a bias vector for the first layer - self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) # create a weight matrix for the second layer - self.bias2=tf.Variable(tf.random.normal([action_dim])) # create a bias vector for the second layer - self.param=[self.weight1,self.bias1,self.weight2,self.bias2] # store the network parameters in a list - - def fp(self,x): # forward propagation function, kernel uses it for forward propagation - x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) # apply the first layer with ReLU activation - return tf.matmul(x,self.weight2)+self.bias2 # apply the second layer and return the output - - -class DQN: # define a class for the DQN agent - def __init__(self,state_dim,hidden_dim,action_dim): # initialize the agent with state dimension, hidden dimension and action dimension - self.nn=Qnet(state_dim,hidden_dim,action_dim) # create a Q-network for the agent - self.target_q_net=Qnet(state_dim,hidden_dim,action_dim) # create a target Q-network for the agent - self.param=self.nn.param # parameter list, kernel uses it list for backpropagation - self.optimizer=o.Momentum(0.07,0.7) # optimizer, kernel uses it to optimize. Here we use a custom momentum optimizer with learning rate 0.07 and momentum 0.7 - self.genv=[gym.make('CartPole-v0') for _ in range(5)] # create a list of 5 environments - - - def env(self,a=None,p=None,initial=None): # environment function, kernel uses it to interact with the environment - if initial==True: # if initial is True, reset the environment with index p and return the initial state - state=self.genv[p].reset(seed=0) - return state - else: # otherwise, take an action and return the next state, reward and done flag from the environment with index p - next_state,reward,done,_=self.genv[p].step(a) - return next_state,reward,done - - - def loss(self,s,a,next_s,r,d): # loss function, kernel uses it to calculate loss - a=tf.expand_dims(a,axis=1) # expand the action vector to match the Q-value matrix shape - q_value=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) # get the Q-value for the selected action - next_q_value=tf.reduce_max(self.target_q_net.fp(next_s),axis=1) # get the maximum Q-value for the next state from the target network - target=r+0.98*next_q_value*(1-d) # calculate the target value using Bellman equation with discount factor 0.98 - return tf.reduce_mean((q_value-target)**2) # return the mean squared error between Q-value and target value - - - def update_param(self): # update function, kernel uses it to update parameter - self.target_q_net.param=self.param.copy() # copy the parameters from the Q-network to the target network - return - - - def opt(self,gradient): # optimization function, kernel uses it to optimize parameter - param=self.optimizer.opt(gradient,self.param) # apply the custom momentum optimizer to update the parameters using the gradient - return param \ No newline at end of file From 9315e5986900356d206919703699f7bce8649469 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Wed, 16 Aug 2023 14:23:07 +0800 Subject: [PATCH 177/337] Add files via upload --- .../DL/pytorch/non_parallel/nn.py | 53 +++++++++ 7.0/neuralnetwork/DL/pytorch/parallel/nn.py | 41 +++++++ .../DL/pytorch/parallel/nn_device.py | 45 ++++++++ 7.0/neuralnetwork/DL/tensorflow/layer/LSTM.py | 37 +++++++ .../DL/tensorflow/layer/attn_LSTM.py | 44 ++++++++ 7.0/neuralnetwork/DL/tensorflow/layer/cnn.py | 50 +++++++++ .../DL/tensorflow/layer/cnn_lmix.py | 58 ++++++++++ 7.0/neuralnetwork/DL/tensorflow/layer/nn.py | 27 +++++ .../DL/tensorflow/layer/nn_acc.py | 32 ++++++ .../DL/tensorflow/layer/nn_clipping.py | 40 +++++++ .../DL/tensorflow/layer/self_LSTM.py | 45 ++++++++ .../DL/tensorflow/layer/transformer.py | 46 ++++++++ .../DL/tensorflow/non_parallel/cnn.py | 27 +++++ .../DL/tensorflow/non_parallel/lstm.py | 26 +++++ .../DL/tensorflow/non_parallel/nn.py | 23 ++++ .../DL/tensorflow/non_parallel/nn_acc.py | 28 +++++ .../DL/tensorflow/non_parallel/nn_ol.py | 37 +++++++ .../DL/tensorflow/parallel/LSTM.py | 46 ++++++++ .../DL/tensorflow/parallel/attn_LSTM.py | 53 +++++++++ .../DL/tensorflow/parallel/cnn.py | 59 ++++++++++ .../DL/tensorflow/parallel/cnn_lmix.py | 67 ++++++++++++ .../DL/tensorflow/parallel/nn.py | 39 +++++++ .../DL/tensorflow/parallel/nn_.py | 39 +++++++ .../DL/tensorflow/parallel/nn_acc.py | 44 ++++++++ .../DL/tensorflow/parallel/nn_attenuate.py | 57 ++++++++++ .../DL/tensorflow/parallel/nn_clipping.py | 47 ++++++++ .../DL/tensorflow/parallel/nn_device.py | 40 +++++++ .../DL/tensorflow/parallel/self_LSTM.py | 54 ++++++++++ .../DL/tensorflow/parallel/transformer.py | 55 ++++++++++ .../RL/pytorch/non_parallel/DDPG.py | 101 ++++++++++++++++++ .../RL/pytorch/non_parallel/DQN.py | 64 +++++++++++ .../RL/pytorch/non_parallel/DQN_pr.py | 76 +++++++++++++ .../RL/pytorch/non_parallel/DoubleDQN.py | 65 +++++++++++ .../RL/pytorch/non_parallel/DuelingDQN.py | 67 ++++++++++++ 7.0/neuralnetwork/RL/pytorch/parallel/DQN.py | 65 +++++++++++ .../RL/tensorflow/non_parallel/DDPG.py | 84 +++++++++++++++ .../RL/tensorflow/non_parallel/DQN.py | 46 ++++++++ .../RL/tensorflow/non_parallel/DQN_pr.py | 59 ++++++++++ .../RL/tensorflow/parallel/DQN.py | 52 +++++++++ 39 files changed, 1938 insertions(+) create mode 100644 7.0/neuralnetwork/DL/pytorch/non_parallel/nn.py create mode 100644 7.0/neuralnetwork/DL/pytorch/parallel/nn.py create mode 100644 7.0/neuralnetwork/DL/pytorch/parallel/nn_device.py create mode 100644 7.0/neuralnetwork/DL/tensorflow/layer/LSTM.py create mode 100644 7.0/neuralnetwork/DL/tensorflow/layer/attn_LSTM.py create mode 100644 7.0/neuralnetwork/DL/tensorflow/layer/cnn.py create mode 100644 7.0/neuralnetwork/DL/tensorflow/layer/cnn_lmix.py create mode 100644 7.0/neuralnetwork/DL/tensorflow/layer/nn.py create mode 100644 7.0/neuralnetwork/DL/tensorflow/layer/nn_acc.py create mode 100644 7.0/neuralnetwork/DL/tensorflow/layer/nn_clipping.py create mode 100644 7.0/neuralnetwork/DL/tensorflow/layer/self_LSTM.py create mode 100644 7.0/neuralnetwork/DL/tensorflow/layer/transformer.py create mode 100644 7.0/neuralnetwork/DL/tensorflow/non_parallel/cnn.py create mode 100644 7.0/neuralnetwork/DL/tensorflow/non_parallel/lstm.py create mode 100644 7.0/neuralnetwork/DL/tensorflow/non_parallel/nn.py create mode 100644 7.0/neuralnetwork/DL/tensorflow/non_parallel/nn_acc.py create mode 100644 7.0/neuralnetwork/DL/tensorflow/non_parallel/nn_ol.py create mode 100644 7.0/neuralnetwork/DL/tensorflow/parallel/LSTM.py create mode 100644 7.0/neuralnetwork/DL/tensorflow/parallel/attn_LSTM.py create mode 100644 7.0/neuralnetwork/DL/tensorflow/parallel/cnn.py create mode 100644 7.0/neuralnetwork/DL/tensorflow/parallel/cnn_lmix.py create mode 100644 7.0/neuralnetwork/DL/tensorflow/parallel/nn.py create mode 100644 7.0/neuralnetwork/DL/tensorflow/parallel/nn_.py create mode 100644 7.0/neuralnetwork/DL/tensorflow/parallel/nn_acc.py create mode 100644 7.0/neuralnetwork/DL/tensorflow/parallel/nn_attenuate.py create mode 100644 7.0/neuralnetwork/DL/tensorflow/parallel/nn_clipping.py create mode 100644 7.0/neuralnetwork/DL/tensorflow/parallel/nn_device.py create mode 100644 7.0/neuralnetwork/DL/tensorflow/parallel/self_LSTM.py create mode 100644 7.0/neuralnetwork/DL/tensorflow/parallel/transformer.py create mode 100644 7.0/neuralnetwork/RL/pytorch/non_parallel/DDPG.py create mode 100644 7.0/neuralnetwork/RL/pytorch/non_parallel/DQN.py create mode 100644 7.0/neuralnetwork/RL/pytorch/non_parallel/DQN_pr.py create mode 100644 7.0/neuralnetwork/RL/pytorch/non_parallel/DoubleDQN.py create mode 100644 7.0/neuralnetwork/RL/pytorch/non_parallel/DuelingDQN.py create mode 100644 7.0/neuralnetwork/RL/pytorch/parallel/DQN.py create mode 100644 7.0/neuralnetwork/RL/tensorflow/non_parallel/DDPG.py create mode 100644 7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN.py create mode 100644 7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN_pr.py create mode 100644 7.0/neuralnetwork/RL/tensorflow/parallel/DQN.py diff --git a/7.0/neuralnetwork/DL/pytorch/non_parallel/nn.py b/7.0/neuralnetwork/DL/pytorch/non_parallel/nn.py new file mode 100644 index 00000000..1faec0fc --- /dev/null +++ b/7.0/neuralnetwork/DL/pytorch/non_parallel/nn.py @@ -0,0 +1,53 @@ +import torch +from torch import nn + + +class NeuralNetwork(nn.Module): + def __init__(self): + super().__init__() + self.flatten=nn.Flatten() + self.linear_relu_stack=nn.Sequential( + nn.Linear(28*28, 512), + nn.ReLU(), + nn.Linear(512, 512), + nn.ReLU(), + nn.Linear(512, 10) + ) + + + def forward(self,x): + x=self.flatten(x) + logits=self.linear_relu_stack(x) + return logits + + +class neuralnetwork: + def __init__(self): + if torch.cuda.is_available(): + self.device=torch.device('cuda') + else: + self.device=torch.device('cpu') + self.model=NeuralNetwork().to(self.device) + self.loss_fn=nn.CrossEntropyLoss() + self.opt=torch.optim.SGD(self.model.parameters(),lr=1e-3) #optimizer,kernel uses it to optimize. + + + def fp(self,x): #forward propagation function,kernel uses it for forward propagation. + pred=self.model(x.to(self.device)) + return pred + + + def loss(self,output,labels): #loss functino,kernel uses it to calculate loss. + loss=self.loss_fn(output,labels.to(self.device)) + return loss + + + def backward(self,loss): #backward function,kernel uses it for backpropagation. + self.optim.zero_grad() + loss.backward() + return + + + def opt(self): #opt function,kernel uses it to optimize. + self.optim.step() + return \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/pytorch/parallel/nn.py b/7.0/neuralnetwork/DL/pytorch/parallel/nn.py new file mode 100644 index 00000000..d9b05567 --- /dev/null +++ b/7.0/neuralnetwork/DL/pytorch/parallel/nn.py @@ -0,0 +1,41 @@ +import torch # import the PyTorch library +from torch import nn # import the neural network module from PyTorch + +class NeuralNetwork(nn.Module): # define a class for the neural network model + def __init__(self): # define the constructor method + super().__init__() # call the parent class constructor + self.flatten = nn.Flatten() # define a layer to flatten the input + self.linear_relu_stack=nn.Sequential( # define a stack of linear and relu layers + nn.Linear(28*28,512), # a linear layer with 28*28 input features and 512 output features + nn.ReLU(), # a relu activation function + nn.Linear(512,512), # another linear layer with 512 input and output features + nn.ReLU(), # another relu activation function + nn.Linear(512,10) # a final linear layer with 512 input features and 10 output features + ) + + + def forward(self,x): # define the forward method + x = self.flatten(x) # flatten the input x + logits=self.linear_relu_stack(x) # pass x through the linear relu stack and get the logits + return logits # return the logits + + +class neuralnetwork: # define another class for the neural network object + def __init__(self): # define the constructor method + if torch.cuda.is_available(): # check if cuda is available + self.device=torch.device('cuda') # set the device to cuda + else: + self.device=torch.device('cpu') # otherwise set the device to cpu + self.model=NeuralNetwork().to(self.device) # create an instance of the NeuralNetwork class and move it to the device + self.loss_fn=nn.CrossEntropyLoss() # define the loss function as cross entropy loss + self.opt=[torch.optim.SGD(self.model.parameters(),lr=1e-3) for _ in range(7)] # define a list of three stochastic gradient descent optimizers with learning rate 1e-3 + + + def fp(self,x): # define a method for forward propagation + pred=self.model(x.to(self.device)) # get the predictions from the model by passing x to the device + return pred # return the predictions + + + def loss(self,output,labels): # define a method for calculating the loss + loss=self.loss_fn(output,labels.to(self.device)) # compute the loss by passing output and labels to the device + return loss # return the loss \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/pytorch/parallel/nn_device.py b/7.0/neuralnetwork/DL/pytorch/parallel/nn_device.py new file mode 100644 index 00000000..ad4b8069 --- /dev/null +++ b/7.0/neuralnetwork/DL/pytorch/parallel/nn_device.py @@ -0,0 +1,45 @@ +import torch # import the PyTorch library +from torch import nn # import the neural network module from PyTorch +from Note.nn.parallel.assign_device_pytorch import assign_device # import a custom function to assign device for PyTorch tensors + + +class NeuralNetwork(nn.Module): # define a class for the neural network model + def __init__(self): # define the constructor method + super().__init__() # call the parent class constructor + self.flatten = nn.Flatten() # define a layer to flatten the input tensor + self.linear_relu_stack=nn.Sequential( # define a stack of linear and relu layers + nn.Linear(28*28,512), # define a linear layer with input size 28*28 and output size 512 + nn.ReLU(), # define a relu activation function + nn.Linear(512,512), # define another linear layer with input and output size 512 + nn.ReLU(), # define another relu activation function + nn.Linear(512,10) # define the final linear layer with output size 10 + ) + + + def forward(self,x): # define the forward method for the model + x = self.flatten(x) # flatten the input tensor + logits=self.linear_relu_stack(x) # pass the flattened tensor through the linear relu stack + return logits # return the logits as the output + + +class neuralnetwork: # define another class for the neural network object + def __init__(self): # define the constructor method + if torch.cuda.is_available(): # check if cuda device is available + self.device=torch.device('cuda') # set the device to cuda + else: # otherwise + self.device=torch.device('cpu') # set the device to cpu + self.model=NeuralNetwork().to(self.device) # create an instance of the neural network model and move it to the device + self.loss_fn=nn.CrossEntropyLoss() # define the loss function as cross entropy loss + self.opt=[torch.optim.SGD(self.model.parameters(),lr=1e-3) for _ in range(7)] # define a list of optimizers as stochastic gradient descent with learning rate 1e-3 + + + def fp(self,x,p): # define a method for forward propagation + pred=self.model(x.to(assign_device(p,'GPU'))) # assign the device according to the process index p + # get the predictions from the model by passing the input tensor to the device and then to the model + return pred # return the predictions + + + def loss(self,output,labels,p): # define a method for calculating loss + loss=self.loss_fn(output,labels.to(assign_device(p,'GPU'))) # assign the device according to the process index p + # calculate the loss by passing the output and labels tensors to the device and then to the loss function + return loss # return the loss \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/layer/LSTM.py b/7.0/neuralnetwork/DL/tensorflow/layer/LSTM.py new file mode 100644 index 00000000..272855a3 --- /dev/null +++ b/7.0/neuralnetwork/DL/tensorflow/layer/LSTM.py @@ -0,0 +1,37 @@ +import tensorflow as tf +from Note.nn.layer.LSTM import LSTM +from Note.nn.layer.dense import dense + +# Define a recurrent neural network class with LSTM layers +class lstm: + def __init__(self): + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.opt=tf.keras.optimizers.Adam() + + + def build(self): + # Create two LSTM layers with 50 hidden units each + self.lstm1=LSTM(weight_shape=(3,50),return_sequence=True) # First LSTM layer that returns the full sequence + self.lstm2=LSTM(weight_shape=(50,50),) # Second LSTM layer that returns the last output only + # Create a dense layer with 10 output units + self.dense=dense([50, 10]) + # Store the parameters of the layers in a list + self.param=[self.lstm1.param, + self.lstm2.param, + self.dense.param, + ] + return + + + def fp(self,data): + # Perform forward propagation on the input data + x=self.lstm1.output(data) # First LSTM layer output + x=self.lstm2.output(x) # Second LSTM layer output + output=self.dense.output(x) # Dense layer output + return output + + + def loss(self, output, labels): + # Compute the loss between the output and the labels + return self.loss_object(labels, output) \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/layer/attn_LSTM.py b/7.0/neuralnetwork/DL/tensorflow/layer/attn_LSTM.py new file mode 100644 index 00000000..1d41befd --- /dev/null +++ b/7.0/neuralnetwork/DL/tensorflow/layer/attn_LSTM.py @@ -0,0 +1,44 @@ +import tensorflow as tf +from Note.nn.layer.LSTM import LSTM +from Note.nn.layer.dense import dense +from Note.nn.layer.attention import attention + +# Define a recurrent neural network class with LSTM layers and attention +class attn_lstm: + def __init__(self): + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.opt=tf.keras.optimizers.Adam() + + + def build(self): + # Create two LSTM layers with 50 hidden units each + self.lstm1=LSTM(weight_shape=(3,50),return_sequence=True) # First LSTM layer that returns the full sequence + self.lstm2=LSTM(weight_shape=(50,50),) # Second LSTM layer that returns the last output only + # Create a dense layer with 10 output units and softmax activation + self.dense=dense([100, 10],activation='softmax') + # Create an attention object with weight shape (50, 50) + self.attn=attention((50,50)) + # Store the parameters of the layers and the attention in a list + self.param=[self.lstm1.param, + self.lstm2.param, + self.dense.param, + self.attn.param] # Add the attention parameters to the list + return + + + def fp(self,data): + # Perform forward propagation on the input data + x=self.lstm1.output(data) # First LSTM layer output, shape (B, N, 50) + y=self.lstm2.output(x) # Second LSTM layer output, shape (B, 50) + # Use the attention object to compute the context vector from x and y, shape (B, 50) + context_vector,_,_=self.attn.output(x,y) + # Concatenate the context vector and the second LSTM output, shape (B, 100) + z=tf.concat([context_vector,y],axis=-1) + output=self.dense.output(z) # Dense layer output with softmax activation, shape (B, 10) + return output + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/layer/cnn.py b/7.0/neuralnetwork/DL/tensorflow/layer/cnn.py new file mode 100644 index 00000000..aa2667f8 --- /dev/null +++ b/7.0/neuralnetwork/DL/tensorflow/layer/cnn.py @@ -0,0 +1,50 @@ +import tensorflow as tf +from Note.nn.layer.conv2d import conv2d +from Note.nn.layer.dense import dense +from Note.nn.layer.flatten import flatten + +# Define a convolutional neural network class +class cnn: + def __init__(self): + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.opt=tf.keras.optimizers.Adam() + + + def build(self): + # Create three convolutional layers with relu activations + self.conv1=conv2d(weight_shape=(3,3,3,32),activation='relu') + self.conv2=conv2d(weight_shape=(3,3,32,64),activation='relu') + self.conv3=conv2d(weight_shape=(3,3,64,64),activation='relu') + # Create two dense layers with relu and linear activations + self.dense1=dense([64*4*4,64],activation='relu') + self.dense2=dense([64,10]) + # Store the parameters of the layers in a list + self.param = [self.conv1.weight, + self.conv2.weight, + self.conv3.weight, + self.dense1.weight, + self.dense1.bias, + self.dense2.weight, + self.dense2.bias] + return + + + def fp(self,data): + # Perform forward propagation on the input data + x=tf.nn.conv2d(data,self.conv1.weight,strides=(1,1),padding='SAME') # First convolutional layer + x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # First max pooling layer + x=tf.nn.conv2d(x,self.conv2.weight,strides=(1,1),padding='SAME') # Second convolutional layer + x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # Second max pooling layer + x=tf.nn.conv2d(x,self.conv3.weight,strides=(1,1),padding='SAME') # Third convolutional layer + x=flatten(x) # Flatten the output to a vector + x=tf.nn.relu(tf.matmul(x,self.dense1.weight)+self.dense1.bias) # First dense layer with relu activation + x=tf.nn.dropout(x,rate=0.5) # Apply dropout to prevent overfitting + output=tf.matmul(x,self.dense2.weight)+self.dense2.bias # Output layer with linear activation + output=tf.nn.dropout(output,rate=0.5) # Apply dropout to prevent overfitting + return output + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/layer/cnn_lmix.py b/7.0/neuralnetwork/DL/tensorflow/layer/cnn_lmix.py new file mode 100644 index 00000000..40232919 --- /dev/null +++ b/7.0/neuralnetwork/DL/tensorflow/layer/cnn_lmix.py @@ -0,0 +1,58 @@ +import tensorflow as tf +from Note.nn.layer.conv2d import conv2d +from Note.nn.layer.dense import dense +from Note.nn.layer.flatten import flatten +from Note.nn.layer.LMix import lmix + +# Define a convolutional neural network class with mixup augmentation +class cnn: + def __init__(self): + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.opt=tf.keras.optimizers.Adam() + # Initialize the alpha parameter for mixup + self.alpha=1.0 + + + def build(self): + # Create three convolutional layers with relu activations + self.conv1=conv2d(weight_shape=(3,3,3,32),activation='relu') + self.conv2=conv2d(weight_shape=(3,3,32,64),activation='relu') + self.conv3=conv2d(weight_shape=(3,3,64,64),activation='relu') + # Create two dense layers with relu and linear activations + self.dense1=dense([64*4*4,64],activation='relu') + self.dense2=dense([64,10]) + # Store the parameters of the layers in a list + self.param = [self.conv1.weight, + self.conv2.weight, + self.conv3.weight, + self.dense1.weight, + self.dense1.bias, + self.dense2.weight, + self.dense2.bias] + return + + + def data_func(self,data_batch,labels_batch): + # Apply mixup augmentation on the input data and labels + return lmix(data_batch,labels_batch,self.alpha) + + + def fp(self,data): + # Perform forward propagation on the input data + x=tf.nn.conv2d(data,self.conv1.weight,strides=(1,1),padding='SAME') # First convolutional layer + x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # First max pooling layer + x=tf.nn.conv2d(x,self.conv2.weight,strides=(1,1),padding='SAME') # Second convolutional layer + x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # Second max pooling layer + x=tf.nn.conv2d(x,self.conv3.weight,strides=(1,1),padding='SAME') # Third convolutional layer + x=flatten(x) # Flatten the output to a vector + x=tf.nn.relu(tf.matmul(x,self.dense1.weight)+self.dense1.bias) # First dense layer with relu activation + x=tf.nn.dropout(x,rate=0.5) # Apply dropout to prevent overfitting + output=tf.matmul(x,self.dense2.weight)+self.dense2.bias # Output layer with linear activation + output=tf.nn.dropout(output,rate=0.5) # Apply dropout to prevent overfitting + return output + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/layer/nn.py b/7.0/neuralnetwork/DL/tensorflow/layer/nn.py new file mode 100644 index 00000000..77ed7454 --- /dev/null +++ b/7.0/neuralnetwork/DL/tensorflow/layer/nn.py @@ -0,0 +1,27 @@ +import tensorflow as tf +import Note.nn.layer.dense as d +from Note.nn.layer.flatten import flatten + + +class nn: + def __init__(self): + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.opt=tf.keras.optimizers.Adam() + + + def build(self): + self.layer1=d.dense([784,128],activation='relu') + self.layer2=d.dense([128,10]) + self.param=[self.layer1.param,self.layer2.param] + return + + + def fp(self,data): + data=flatten(data) + output1=self.layer1.output(data) + output2=self.layer2.output(output1) + return output2 + + + def loss(self,output,labels): + return self.loss_object(labels,output) \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/layer/nn_acc.py b/7.0/neuralnetwork/DL/tensorflow/layer/nn_acc.py new file mode 100644 index 00000000..cad20961 --- /dev/null +++ b/7.0/neuralnetwork/DL/tensorflow/layer/nn_acc.py @@ -0,0 +1,32 @@ +import tensorflow as tf +import Note.nn.layer.dense as d +from Note.nn.layer.flatten import flatten + + +class nn: + def __init__(self): + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.train_accuracy=tf.keras.metrics.SparseCategoricalAccuracy(name='train_accuracy') + self.opt=tf.keras.optimizers.Adam() + + + def build(self): + self.layer1=d.dense([784,128],activation='relu') + self.layer2=d.dense([128,10]) + self.param=[self.layer1.param,self.layer2.param] + return + + + def fp(self,data): + data=flatten(data) + output1=self.layer1.output(data) + output2=self.layer2.output(output1) + return output2 + + + def loss(self,output,labels): + return self.loss_object(labels,output) + + + def accuracy(self,output,labels): + return self.train_accuracy(labels,output) \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/layer/nn_clipping.py b/7.0/neuralnetwork/DL/tensorflow/layer/nn_clipping.py new file mode 100644 index 00000000..b53a43da --- /dev/null +++ b/7.0/neuralnetwork/DL/tensorflow/layer/nn_clipping.py @@ -0,0 +1,40 @@ +import tensorflow as tf +import Note.nn.layer.dense as d +from Note.nn.layer.flatten import flatten +from Note.nn.gradient_clipping import gradient_clipping + +# Define a neural network class +class nn: + def __init__(self): + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.opt=tf.keras.optimizers.Adam() + + + def build(self): + # Create two dense layers with relu and linear activations + self.layer1=d.dense([784,128],activation='relu') + self.layer2=d.dense([128,10]) + # Store the parameters of the layers in a list + self.param=[self.layer1.param,self.layer2.param] + return + + + def fp(self,data): + # Perform forward propagation on the input data + data=flatten(data) + output1=self.layer1.output(data) + output2=self.layer2.output(output1) + return output2 + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) + + + def gradient(self,tape,loss): + # Compute the gradients of the loss with respect to the parameters + grads=tape.gradient(loss,self.param) + # Clip the gradients by value to avoid exploding gradients + return gradient_clipping(grads,'value',1.0) \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/layer/self_LSTM.py b/7.0/neuralnetwork/DL/tensorflow/layer/self_LSTM.py new file mode 100644 index 00000000..edaa2420 --- /dev/null +++ b/7.0/neuralnetwork/DL/tensorflow/layer/self_LSTM.py @@ -0,0 +1,45 @@ +import tensorflow as tf +from Note.nn.layer.LSTM import LSTM +from Note.nn.layer.self_attention import self_attention +from Note.nn.layer.dense import dense +from Note.nn.layer.emdedding import embedding + +# Define a LSTM self attention neural network class +class self_LSTM: + def __init__(self,vocab_size,embed_size,num_heads,num_layers): + self.vocab_size=vocab_size + self.embed_size=embed_size + self.num_heads=num_heads + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.opt=tf.keras.optimizers.Adam() + + + def build(self): + # Create an embedding layer to map input tokens to vectors + self.embedding_layer = embedding(self.vocab_size,self.embed_size) + # Create a LSTM layer to process the input sequence + self.lstm_layer = LSTM(weight_shape=[self.embed_size,self.embed_size],return_sequence=True) + # Create a self attention layer to attend to the LSTM output + self.self_attention_layer = self_attention(weight_shape=[self.embed_size,self.embed_size],num_heads=self.num_heads) + # Create a linear layer to map the attention output to logits + self.logits_layer = dense([self.embed_size,self.vocab_size]) + # Store the parameters of the layers in a list + self.param = [self.embedding_layer.embeddings] + self.param.extend(self.lstm_layer.param) + self.param.extend(self.self_attention_layer.param) + self.param.extend(self.logits_layer.param) + + + def fp(self,data): + # Perform forward propagation on the input data + x = self.embedding_layer.output(data) # shape: (batch_size, seq_len, embed_size) + x = self.lstm_layer.output(x) # shape: (batch_size, seq_len, embed_size) + x, attention_weights = self.self_attention_layer.output(x,self.num_heads) # shape: (batch_size, seq_len, embed_size) + output = self.logits_layer.output(x) # shape: (batch_size, seq_len, vocab_size) + return output + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/layer/transformer.py b/7.0/neuralnetwork/DL/tensorflow/layer/transformer.py new file mode 100644 index 00000000..47862006 --- /dev/null +++ b/7.0/neuralnetwork/DL/tensorflow/layer/transformer.py @@ -0,0 +1,46 @@ +import tensorflow as tf +from Note.nn.layer.transformer import transformer +from Note.nn.layer.dense import dense +from Note.nn.layer.emdedding import embedding +from Note.nn.positional_encoding import positional_encoding + +# Define a positional encoding transformer neural network class +class Transformer: + def __init__(self,vocab_size,embed_size,num_heads,num_layers,max_len): + self.vocab_size=vocab_size + self.embed_size=embed_size + self.num_heads=num_heads + self.num_layers=num_layers + self.max_len=max_len + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.opt=tf.keras.optimizers.Adam() + + + def build(self): + # Create an embedding layer to map input tokens to vectors + self.embedding_layer = embedding(self.vocab_size,self.embed_size) + # Create a list of transformer layers + self.transformer_layers = [transformer(weight_shape=[self.embed_size, self.embed_size], num_heads=self.num_heads) for _ in range(self.num_layers)] + # Create a linear layer to map the transformer output to logits + self.logits_layer = dense([self.embed_size,self.vocab_size]) + # Store the parameters of the layers in a list + self.param = [self.embedding_layer.embeddings] + for transformer_layer in self.transformer_layers: + self.param.extend(transformer_layer.param) + self.param.extend(self.logits_layer.param) + + + def fp(self,data): + # Perform forward propagation on the input data + x = self.embedding_layer.output(data) # shape: (batch_size, seq_len, embed_size) + x += positional_encoding(self.max_len,self.embed_size) # shape: (batch_size, seq_len, embed_size) + for transformer_layer in self.transformer_layers: + x = transformer_layer.output(x) # shape: (batch_size, seq_len, embed_size) + output = self.logits_layer.output(x) # shape: (batch_size, seq_len, vocab_size) + return output + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/non_parallel/cnn.py b/7.0/neuralnetwork/DL/tensorflow/non_parallel/cnn.py new file mode 100644 index 00000000..bd7ad1bf --- /dev/null +++ b/7.0/neuralnetwork/DL/tensorflow/non_parallel/cnn.py @@ -0,0 +1,27 @@ +import tensorflow as tf +from tensorflow.keras import layers,models + + +class cnn: + def __init__(self): + self.model=models.Sequential() + self.model.add(layers.Conv2D(32,(3,3),activation='relu',input_shape=(32,32,3))) + self.model.add(layers.MaxPooling2D((2,2))) + self.model.add(layers.Conv2D(64,(3,3),activation='relu')) + self.model.add(layers.MaxPooling2D((2,2))) + self.model.add(layers.Conv2D(64,(3,3),activation='relu')) + self.model.add(layers.Flatten()) + self.model.add(layers.Dense(64,activation='relu')) + self.model.add(layers.Dense(10)) + self.param=self.model.weights + self.opt=tf.keras.optimizers.Adam() + + + def fp(self,data): + output=self.model(data) + return output + + + def loss(self,output,labels): + loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + return loss(labels,output) \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/non_parallel/lstm.py b/7.0/neuralnetwork/DL/tensorflow/non_parallel/lstm.py new file mode 100644 index 00000000..29b9b396 --- /dev/null +++ b/7.0/neuralnetwork/DL/tensorflow/non_parallel/lstm.py @@ -0,0 +1,26 @@ +import tensorflow as tf + + +class lstm: + def __init__(self,encoder): + self.model=tf.keras.Sequential([ + encoder, + tf.keras.layers.Embedding(len(encoder.get_vocabulary()), 64, mask_zero=True), + tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(64, return_sequences=True)), + tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(32)), + tf.keras.layers.Dense(64, activation='relu'), + tf.keras.layers.Dropout(0.5), + tf.keras.layers.Dense(1) + ]) + self.param=self.model.weights[1:] + self.opt=tf.keras.optimizers.Adam() + + + def fp(self,data): + output=self.model(data) + return output + + + def loss(self,output,labels): + bce=tf.keras.losses.BinaryCrossentropy(from_logits=True) + return bce(labels,output) \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/non_parallel/nn.py b/7.0/neuralnetwork/DL/tensorflow/non_parallel/nn.py new file mode 100644 index 00000000..b4f03a92 --- /dev/null +++ b/7.0/neuralnetwork/DL/tensorflow/non_parallel/nn.py @@ -0,0 +1,23 @@ +import tensorflow as tf + + +class nn: + def __init__(self): + self.model=tf.keras.models.Sequential([ + tf.keras.layers.Flatten(input_shape=(28, 28)), + tf.keras.layers.Dense(128,activation='relu'), + tf.keras.layers.Dropout(0.2), + tf.keras.layers.Dense(10) + ]) + self.param=self.model.weights #parameter list,kernel uses it list for backpropagation. + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.opt=tf.keras.optimizers.Adam() #optimizer,kernel uses it to optimize. + + + def fp(self,data): #forward propagation function,kernel uses it for forward propagation. + output=self.model(data) + return output + + + def loss(self,output,labels): #loss functino,kernel uses it to calculate loss. + return self.loss_object(labels,output) \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/non_parallel/nn_acc.py b/7.0/neuralnetwork/DL/tensorflow/non_parallel/nn_acc.py new file mode 100644 index 00000000..0a726603 --- /dev/null +++ b/7.0/neuralnetwork/DL/tensorflow/non_parallel/nn_acc.py @@ -0,0 +1,28 @@ +import tensorflow as tf + +#An example with accuracy function. +class nn: + def __init__(self): + self.model=tf.keras.models.Sequential([ + tf.keras.layers.Flatten(input_shape=(28, 28)), + tf.keras.layers.Dense(128,activation='relu'), + tf.keras.layers.Dropout(0.2), + tf.keras.layers.Dense(10) + ]) + self.param=self.model.weights #parameter list,kernel uses it list for backpropagation. + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.train_accuracy=tf.keras.metrics.SparseCategoricalAccuracy(name='train_accuracy') + self.opt=tf.keras.optimizers.Adam() #optimizer,kernel uses it to optimize. + + + def fp(self,data): #forward propagation function,kernel uses it for forward propagation. + output=self.model(data) + return output + + + def loss(self,output,labels): #loss functino,kernel uses it to calculate loss. + return self.loss_object(labels,output) + + + def accuracy(self,output,labels): #accuracy function,kernel uses it to calculate accuracy. + return self.train_accuracy(labels,output) \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/non_parallel/nn_ol.py b/7.0/neuralnetwork/DL/tensorflow/non_parallel/nn_ol.py new file mode 100644 index 00000000..9dfa608d --- /dev/null +++ b/7.0/neuralnetwork/DL/tensorflow/non_parallel/nn_ol.py @@ -0,0 +1,37 @@ +import tensorflow as tf # import TensorFlow library +import numpy as np # import NumPy library + +# This is an example of online training. +class nn: # define a class for the neural network + def __init__(self,data,labels): # initialize the network with data and labels + self.train_data=data # store the training data + self.train_labels=labels # store the training labels + self.model=tf.keras.models.Sequential([ # create a sequential model with four layers + tf.keras.layers.Flatten(input_shape=(28, 28)), # flatten the input to a vector of 28*28 elements + tf.keras.layers.Dense(128,activation='relu'), # add a dense layer with 128 units and ReLU activation + tf.keras.layers.Dropout(0.2), # add a dropout layer with 0.2 dropout rate to prevent overfitting + tf.keras.layers.Dense(10) # add a dense layer with 10 units for the output logits + ]) + self.param=self.model.weights # parameter list, kernel uses it list for backpropagation + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) # loss object, kernel uses it to calculate loss. Here we use sparse categorical crossentropy with logits as output + self.opt=tf.keras.optimizers.Adam() # optimizer, kernel uses it to optimize. Here we use Adam optimizer + self.train_loss_list=[] # a list to store the training loss values + self.counter=0 # counter to keep track of the number of online updates + self.max_length=1000 # maximum length of train_loss_list + + + def online(self): # This is simulative online function, kernel uses it to get online data and labels + index=np.random.choice(60000,size=[32]) # randomly sample 32 indices from 0 to 59999 + if self.counter==10000: # if the counter reaches 10000, stop the online training + return 'stop' + else: # otherwise, return the data and labels corresponding to the sampled indices + return [self.train_data[index],self.train_labels[index]] + + + def fp(self,data): # forward propagation function, kernel uses it for forward propagation + output=self.model(data) # pass the data through the model and get the output logits + return output + + + def loss(self,output,labels): # loss function, kernel uses it to calculate loss + return self.loss_object(labels,output) # return the sparse categorical crossentropy loss between labels and output \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/LSTM.py b/7.0/neuralnetwork/DL/tensorflow/parallel/LSTM.py new file mode 100644 index 00000000..e8686c10 --- /dev/null +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/LSTM.py @@ -0,0 +1,46 @@ +import tensorflow as tf +from Note.nn.layer.LSTM import LSTM +from Note.nn.layer.dense import dense +from Note.nn.parallel.optimizer import Adam + +# Define a recurrent neural network class with LSTM layers +class lstm: + def __init__(self): + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.optimizer=Adam() + # Initialize a variable to keep track of the batch count + self.bc=tf.Variable(0, dtype=tf.float32) + + + def build(self): + # Create two LSTM layers with 50 hidden units each + self.lstm1=LSTM(weight_shape=(3,50),return_sequence=True) # First LSTM layer that returns the full sequence + self.lstm2=LSTM(weight_shape=(50,50),) # Second LSTM layer that returns the last output only + # Create a dense layer with 10 output units + self.dense=dense([50, 10]) + # Store the parameters of the layers in a list + self.param=[self.lstm1.param, + self.lstm2.param, + self.dense.param, + ] + return + + + def fp(self,data): + # Perform forward propagation on the input data + x=self.lstm1.output(data) # First LSTM layer output + x=self.lstm2.output(x) # Second LSTM layer output + output=self.dense.output(x) # Dense layer output + return output + + + def loss(self, output, labels): + # Compute the loss between the output and the labels + return self.loss_object(labels, output) + + + def opt(self,gradient): + # Perform optimization on the parameters using the gradient and the batch count + param=self.optimizer.opt(gradient,self.param,self.bc[0]) + return param \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/attn_LSTM.py b/7.0/neuralnetwork/DL/tensorflow/parallel/attn_LSTM.py new file mode 100644 index 00000000..198e46b1 --- /dev/null +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/attn_LSTM.py @@ -0,0 +1,53 @@ +import tensorflow as tf +from Note.nn.layer.LSTM import LSTM +from Note.nn.layer.dense import dense +from Note.nn.layer.attention import attention +from Note.nn.parallel.optimizer import Adam + +# Define a recurrent neural network class with LSTM layers and attention +class attn_lstm: + def __init__(self): + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.optimizer=Adam() + # Initialize a variable to keep track of the batch count + self.bc=tf.Variable(0, dtype=tf.float32) + + + def build(self): + # Create two LSTM layers with 50 hidden units each + self.lstm1=LSTM(weight_shape=(3,50),return_sequence=True) # First LSTM layer that returns the full sequence + self.lstm2=LSTM(weight_shape=(50,50),) # Second LSTM layer that returns the last output only + # Create a dense layer with 10 output units and softmax activation + self.dense=dense([100, 10],activation='softmax') + # Create an attention object with weight shape (50, 50) + self.attn=attention((50,50)) + # Store the parameters of the layers and the attention in a list + self.param=[self.lstm1.param, + self.lstm2.param, + self.dense.param, + self.attn.param] # Add the attention parameters to the list + return + + + def fp(self,data): + # Perform forward propagation on the input data + x=self.lstm1.output(data) # First LSTM layer output, shape (B, N, 50) + y=self.lstm2.output(x) # Second LSTM layer output, shape (B, 50) + # Use the attention object to compute the context vector from x and y, shape (B, 50) + context_vector,_,_=self.attn.output(x,y) + # Concatenate the context vector and the second LSTM output, shape (B, 100) + z=tf.concat([context_vector,y],axis=-1) + output=self.dense.output(z) # Dense layer output with softmax activation, shape (B, 10) + return output + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) + + + def opt(self,gradient): + # Perform optimization on the parameters using the gradient and the batch count + param=self.optimizer.opt(gradient,self.param,self.bc[0]) + return param \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/cnn.py b/7.0/neuralnetwork/DL/tensorflow/parallel/cnn.py new file mode 100644 index 00000000..59be2e63 --- /dev/null +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/cnn.py @@ -0,0 +1,59 @@ +import tensorflow as tf +from Note.nn.layer.conv2d import conv2d +from Note.nn.layer.dense import dense +from Note.nn.layer.flatten import flatten +from Note.nn.parallel.optimizer import Adam + +# Define a convolutional neural network class +class cnn: + def __init__(self): + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.optimizer=Adam() + # Initialize a variable to keep track of the batch count + self.bc=tf.Variable(0,dtype=tf.float32) + + + def build(self): + # Create three convolutional layers with relu activations + self.conv1=conv2d(weight_shape=(3,3,3,32),activation='relu') + self.conv2=conv2d(weight_shape=(3,3,32,64),activation='relu') + self.conv3=conv2d(weight_shape=(3,3,64,64),activation='relu') + # Create two dense layers with relu and linear activations + self.dense1=dense([64*4*4,64],activation='relu') + self.dense2=dense([64,10]) + # Store the parameters of the layers in a list + self.param = [self.conv1.weight, + self.conv2.weight, + self.conv3.weight, + self.dense1.weight, + self.dense1.bias, + self.dense2.weight, + self.dense2.bias] + return + + + def fp(self,data): + # Perform forward propagation on the input data + x=tf.nn.conv2d(data,self.conv1.weight,strides=(1,1),padding='SAME') # First convolutional layer + x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # First max pooling layer + x=tf.nn.conv2d(x,self.conv2.weight,strides=(1,1),padding='SAME') # Second convolutional layer + x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # Second max pooling layer + x=tf.nn.conv2d(x,self.conv3.weight,strides=(1,1),padding='SAME') # Third convolutional layer + x=flatten(x) # Flatten the output to a vector + x=tf.nn.relu(tf.matmul(x,self.dense1.weight)+self.dense1.bias) # First dense layer with relu activation + x=tf.nn.dropout(x,rate=0.5) # Apply dropout to prevent overfitting + output=tf.matmul(x,self.dense2.weight)+self.dense2.bias # Output layer with linear activation + output=tf.nn.dropout(output,rate=0.5) # Apply dropout to prevent overfitting + return output + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) + + + def opt(self,gradient): + # Perform optimization on the parameters using the gradient and the batch count + param=self.optimizer.opt(gradient,self.param,self.bc[0]) + return param \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/cnn_lmix.py b/7.0/neuralnetwork/DL/tensorflow/parallel/cnn_lmix.py new file mode 100644 index 00000000..b6e66458 --- /dev/null +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/cnn_lmix.py @@ -0,0 +1,67 @@ +import tensorflow as tf +from Note.nn.layer.conv2d import conv2d +from Note.nn.layer.dense import dense +from Note.nn.layer.flatten import flatten +from Note.nn.parallel.optimizer import Adam +from Note.nn.layer.LMix import lmix + +# Define a convolutional neural network class with mixup augmentation +class cnn: + def __init__(self): + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.optimizer=Adam() + # Initialize the alpha parameter for mixup + self.alpha=1.0 + # Initialize a variable to keep track of the batch count + self.bc=tf.Variable(0,dtype=tf.float32) + + + def build(self): + # Create three convolutional layers with relu activations + self.conv1=conv2d(weight_shape=(3,3,3,32),activation='relu') + self.conv2=conv2d(weight_shape=(3,3,32,64),activation='relu') + self.conv3=conv2d(weight_shape=(3,3,64,64),activation='relu') + # Create two dense layers with relu and linear activations + self.dense1=dense([64*4*4,64],activation='relu') + self.dense2=dense([64,10]) + # Store the parameters of the layers in a list + self.param = [self.conv1.weight, + self.conv2.weight, + self.conv3.weight, + self.dense1.weight, + self.dense1.bias, + self.dense2.weight, + self.dense2.bias] + return + + + def data_func(self,data_batch,labels_batch): + # Apply mixup augmentation on the input data and labels + return lmix(data_batch,labels_batch,self.alpha) + + + def fp(self,data): + # Perform forward propagation on the input data + x=tf.nn.conv2d(data,self.conv1.weight,strides=(1,1),padding='SAME') # First convolutional layer + x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # First max pooling layer + x=tf.nn.conv2d(x,self.conv2.weight,strides=(1,1),padding='SAME') # Second convolutional layer + x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # Second max pooling layer + x=tf.nn.conv2d(x,self.conv3.weight,strides=(1,1),padding='SAME') # Third convolutional layer + x=flatten(x) # Flatten the output to a vector + x=tf.nn.relu(tf.matmul(x,self.dense1.weight)+self.dense1.bias) # First dense layer with relu activation + x=tf.nn.dropout(x,rate=0.5) # Apply dropout to prevent overfitting + output=tf.matmul(x,self.dense2.weight)+self.dense2.bias # Output layer with linear activation + output=tf.nn.dropout(output,rate=0.5) # Apply dropout to prevent overfitting + return output + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) + + + def opt(self,gradient): + # Perform optimization on the parameters using the gradient and the batch count + param=self.optimizer.opt(gradient,self.param,self.bc[0]) + return param \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn.py new file mode 100644 index 00000000..1b16b5c0 --- /dev/null +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/nn.py @@ -0,0 +1,39 @@ +import tensorflow as tf +import Note.nn.layer.dense as d +from Note.nn.parallel.optimizer import Momentum +from Note.nn.layer.flatten import flatten + +# Define a neural network class +class nn: + def __init__(self): + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.optimizer=Momentum(0.07,0.7) + + + def build(self): + # Create two dense layers with relu and linear activations + self.layer1=d.dense([784,128],activation='relu') + self.layer2=d.dense([128,10]) + # Store the parameters of the layers in a list + self.param=[self.layer1.param,self.layer2.param] + return + + + def fp(self,data): + # Perform forward propagation on the input data + data=flatten(data) + output1=self.layer1.output(data) + output2=self.layer2.output(output1) + return output2 + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) + + + def opt(self,gradient): + # Perform optimization on the parameters using the gradient + param=self.optimizer.opt(gradient,self.param) + return param \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_.py new file mode 100644 index 00000000..d115cc0c --- /dev/null +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_.py @@ -0,0 +1,39 @@ +import tensorflow as tf +from Note.nn.parallel.optimizer import Momentum +from Note.nn.layer.flatten import flatten + +# Define a neural network class +class nn: + def __init__(self): + # Initialize the weights and biases of three layers with random values + self.weight1=tf.Variable(tf.random.normal([784,64],dtype='float64')) + self.bias1=tf.Variable(tf.random.normal([64],dtype='float64')) + self.weight2=tf.Variable(tf.random.normal([64,64],dtype='float64')) + self.bias2=tf.Variable(tf.random.normal([64],dtype='float64')) + self.weight3=tf.Variable(tf.random.normal([64,10],dtype='float64')) + self.bias3=tf.Variable(tf.random.normal([10],dtype='float64')) + # Store the parameters of the layers in a list + self.param=[self.weight1,self.weight2,self.weight3,self.bias1,self.bias2,self.bias3] + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.optimizer=Momentum(0.07,0.7) + + + def fp(self,data): + # Perform forward propagation on the input data + data=flatten(data) + layer1=tf.nn.relu(tf.matmul(data,self.weight1)+self.bias1) # First layer with relu activation + layer2=tf.nn.relu(tf.matmul(layer1,self.weight2)+self.bias2) # Second layer with relu activation + output=tf.matmul(layer2,self.weight3)+self.bias3 # Output layer with linear activation + return output + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) + + + def opt(self,gradient): + # Perform optimization on the parameters using the gradient + param=self.optimizer.opt(gradient,self.param) + return param \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_acc.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_acc.py new file mode 100644 index 00000000..19b055dc --- /dev/null +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_acc.py @@ -0,0 +1,44 @@ +import tensorflow as tf +import Note.nn.layer.dense as d +from Note.nn.parallel.optimizer import Momentum +from Note.nn.layer.flatten import flatten +from Note.nn.accuracy import sparse_categorical_accuracy + +# Define a neural network class +class nn: + def __init__(self): + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.optimizer=Momentum(0.07,0.7) + + + def build(self): + # Create two dense layers with relu and linear activations + self.layer1=d.dense([784,128],activation='relu') + self.layer2=d.dense([128,10]) + # Store the parameters of the layers in a list + self.param=[self.layer1.param,self.layer2.param] + return + + + def fp(self,data): + # Perform forward propagation on the input data + data=flatten(data) + output1=self.layer1.output(data) + output2=self.layer2.output(output1) + return output2 + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) + + + def accuracy(self,output,labels): #accuracy function,kernel uses it to calculate accuracy. + return sparse_categorical_accuracy(labels,output) + + + def opt(self,gradient): + # Perform optimization on the parameters using the gradient + param=self.optimizer.opt(gradient,self.param) + return param \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_attenuate.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_attenuate.py new file mode 100644 index 00000000..2f176151 --- /dev/null +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_attenuate.py @@ -0,0 +1,57 @@ +import tensorflow as tf +from tensorflow.python.util import nest +import Note.nn.layer.dense as d +from Note.nn.parallel.optimizer import Momentum +from Note.nn.layer.flatten import flatten +from Note.nn.accuracy import sparse_categorical_accuracy + +# Define a neural network class with gradient attenuation +class nn: + def __init__(self): + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.optimizer=Momentum(0.07,0.7) + # Initialize a variable to keep track of the number of optimization steps + self.opt_counter=tf.Variable(tf.zeros(3,dtype=tf.float32)) + + + def build(self): + # Create two dense layers with relu and linear activations + self.layer1=d.dense([784,128],activation='relu') + self.layer2=d.dense([128,10]) + # Store the parameters of the layers in a list + self.param=[self.layer1.param,self.layer2.param] + return + + + def fp(self,data): + # Perform forward propagation on the input data + data=flatten(data) # Flatten the data to a one-dimensional vector + output1=self.layer1.output(data) # Apply the first layer to the data and get the output + output2=self.layer2.output(output1) # Apply the second layer to the output of the first layer and get the final output + return output2 + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) # Use the loss function to calculate the loss + + + def accuracy(self,output,labels): #accuracy function,kernel uses it to calculate accuracy. + return sparse_categorical_accuracy(labels,output) + + + def attenuate(self,gradient,oc,p): #gradient attenuation function,kernel uses it to calculate attenuation coefficient. + # Apply an exponential decay to the gradient based on the optimization counter + ac=0.9**oc[0][p] #ac:attenuation coefficient + gradient_flat=nest.flatten(gradient) # Flatten the gradient to a one-dimensional vector + for i in range(len(gradient_flat)): #oc:optimization counter + gradient_flat[i]=ac*gradient_flat[i] #p:process number + gradient=nest.pack_sequence_as(gradient,gradient_flat) # Restore the gradient to its original shape + return gradient + + + def opt(self,gradient): + # Perform optimization on the parameters using the gradient + param=self.optimizer.opt(gradient,self.param) # Use the optimizer to update the parameters + return param \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_clipping.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_clipping.py new file mode 100644 index 00000000..a0f757d3 --- /dev/null +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_clipping.py @@ -0,0 +1,47 @@ +import tensorflow as tf +import Note.nn.layer.dense as d +from Note.nn.parallel.optimizer import Momentum +from Note.nn.layer.flatten import flatten +from Note.nn.gradient_clipping import gradient_clipping + +# Define a neural network class +class nn: + def __init__(self): + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.optimizer=Momentum(0.07,0.7) + + + def build(self): + # Create two dense layers with relu and linear activations + self.layer1=d.dense([784,128],activation='relu') + self.layer2=d.dense([128,10]) + # Store the parameters of the layers in a list + self.param=[self.layer1.param,self.layer2.param] + return + + + def fp(self,data): + # Perform forward propagation on the input data + data=flatten(data) + output1=self.layer1.output(data) + output2=self.layer2.output(output1) + return output2 + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) + + + def gradient(self,tape,loss): + # Compute the gradients of the loss with respect to the parameters + grads=tape.gradient(loss,self.param) + # Clip the gradients by value to avoid exploding gradients + return gradient_clipping(grads,'value',1.0) + + + def opt(self,gradient): + # Perform optimization on the parameters using the gradient + param=self.optimizer.opt(gradient,self.param) + return param \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_device.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_device.py new file mode 100644 index 00000000..e48acac3 --- /dev/null +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_device.py @@ -0,0 +1,40 @@ +import tensorflow as tf # import TensorFlow library +import Note.nn.layer.dense as d # import Note's dense layer module +from Note.nn.layer.flatten import flatten # import Note's flatten layer function +from Note.nn.parallel.optimizer import Momentum # import Note's momentum optimizer module +from Note.nn.parallel.assign_device import assign_device # import the function to assign device according to the process index and the device type + + +class nn: # A neural network class example, allocate device for multiple threads + def __init__(self): # initialize the network + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) # loss object, kernel uses it to calculate loss. Here we use sparse categorical crossentropy with logits as output + self.optimizer=Momentum(0.07,0.7) # optimizer, kernel uses it to optimize. Here we use a custom momentum optimizer with learning rate 0.07 and momentum 0.7 + + + def build(self): # build function, kernel uses it to create the network layers + # Create two dense layers with relu and linear activations + self.layer1=d.dense([784,128],activation='relu') # the first layer with 784 input units and 128 output units and ReLU activation + self.layer2=d.dense([128,10]) # the second layer with 128 input units and 10 output units and linear activation + # Store the parameters of the layers in a list + self.param=[self.layer1.param,self.layer2.param] # parameter list of both layers, kernel uses it list for backpropagation + return + + + def fp(self,data,p): # forward propagation function, kernel uses it for forward propagation + with tf.device(assign_device(p,'GPU')): # assign the device according to the process index p + data=flatten(data) # flatten the data to a vector of 784 elements + output1=self.layer1.output(data) # pass the data through the first layer and get the output + output2=self.layer2.output(output1) # pass the output of the first layer through the second layer and get the final output logits + return output2 + + + def loss(self,output,labels,p): # loss function, kernel uses it to calculate loss + with tf.device(assign_device(p,'GPU')): # assign the device according to the process index p + return self.loss_object(labels,output) # return the mean softmax cross entropy loss between labels and output + + + def opt(self,gradient,p): # optimization function, kernel uses it to optimize parameter + # Perform optimization on the parameters using the gradient + with tf.device(assign_device(p,'GPU')): # assign the device according to the process index p + param=self.optimizer.opt(gradient,self.param) # apply the Note's momentum optimizer to update the parameters using the gradient + return param \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/self_LSTM.py b/7.0/neuralnetwork/DL/tensorflow/parallel/self_LSTM.py new file mode 100644 index 00000000..85af312a --- /dev/null +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/self_LSTM.py @@ -0,0 +1,54 @@ +import tensorflow as tf +from Note.nn.parallel.optimizer import Adam +from Note.nn.layer.LSTM import LSTM +from Note.nn.layer.self_attention import self_attention +from Note.nn.layer.dense import dense +from Note.nn.layer.emdedding import embedding + +# Define a LSTM self attention neural network class +class self_LSTM: + def __init__(self,vocab_size,embed_size,num_heads,num_layers): + self.vocab_size=vocab_size + self.embed_size=embed_size + self.num_heads=num_heads + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.optimizer=Adam() + # Initialize a variable to keep track of the batch count + self.bc=tf.Variable(0,dtype=tf.float32) + + + def build(self): + # Create an embedding layer to map input tokens to vectors + self.embedding_layer = embedding(self.vocab_size,self.embed_size) + # Create a LSTM layer to process the input sequence + self.lstm_layer = LSTM(weight_shape=[self.embed_size,self.embed_size],return_sequence=True) + # Create a self attention layer to attend to the LSTM output + self.self_attention_layer = self_attention(weight_shape=[self.embed_size,self.embed_size],num_heads=self.num_heads) + # Create a linear layer to map the attention output to logits + self.logits_layer = dense([self.embed_size,self.vocab_size]) + # Store the parameters of the layers in a list + self.param = [self.embedding_layer.embeddings] + self.param.extend(self.lstm_layer.param) + self.param.extend(self.self_attention_layer.param) + self.param.extend(self.logits_layer.param) + + + def fp(self,data): + # Perform forward propagation on the input data + x = self.embedding_layer.output(data) # shape: (batch_size, seq_len, embed_size) + x = self.lstm_layer.output(x) # shape: (batch_size, seq_len, embed_size) + x, attention_weights = self.self_attention_layer.output(x,self.num_heads) # shape: (batch_size, seq_len, embed_size) + output = self.logits_layer.output(x) # shape: (batch_size, seq_len, vocab_size) + return output + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) + + + def opt(self,gradient): + # Perform optimization on the parameters using the gradient and the batch count + param=self.optimizer.opt(gradient,self.param,self.bc[0]) + return param \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/transformer.py b/7.0/neuralnetwork/DL/tensorflow/parallel/transformer.py new file mode 100644 index 00000000..ae052538 --- /dev/null +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/transformer.py @@ -0,0 +1,55 @@ +import tensorflow as tf +from Note.nn.parallel.optimizer import Adam +from Note.nn.layer.transformer import transformer +from Note.nn.layer.dense import dense +from Note.nn.layer.emdedding import embedding +from Note.nn.positional_encoding import positional_encoding + +# Define a positional encoding transformer neural network class +class Transformer: + def __init__(self,vocab_size,embed_size,num_heads,num_layers,max_len): + self.vocab_size=vocab_size + self.embed_size=embed_size + self.num_heads=num_heads + self.num_layers=num_layers + self.max_len=max_len + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.optimizer=Adam() + # Initialize a variable to keep track of the batch count + self.bc=tf.Variable(0,dtype=tf.float32) + + + def build(self): + # Create an embedding layer to map input tokens to vectors + self.embedding_layer = embedding(self.vocab_size,self.embed_size) + # Create a list of transformer layers + self.transformer_layers = [transformer(weight_shape=[self.embed_size, self.embed_size], num_heads=self.num_heads) for _ in range(self.num_layers)] + # Create a linear layer to map the transformer output to logits + self.logits_layer = dense([self.embed_size,self.vocab_size]) + # Store the parameters of the layers in a list + self.param = [self.embedding_layer.embeddings] + for transformer_layer in self.transformer_layers: + self.param.extend(transformer_layer.param) + self.param.extend(self.logits_layer.param) + + + def fp(self,data): + # Perform forward propagation on the input data + x = self.embedding_layer.output(data) # shape: (batch_size, seq_len, embed_size) + x += positional_encoding(self.max_len,self.embed_size) # shape: (batch_size, seq_len, embed_size) + for transformer_layer in self.transformer_layers: + x = transformer_layer.output(x) # shape: (batch_size, seq_len, embed_size) + output = self.logits_layer.output(x) # shape: (batch_size, seq_len, vocab_size) + return output + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) + + + def opt(self,gradient): + # Perform optimization on the parameters using the gradient and the batch count + param=self.optimizer.opt(gradient,self.param,self.bc[0]) + return param \ No newline at end of file diff --git a/7.0/neuralnetwork/RL/pytorch/non_parallel/DDPG.py b/7.0/neuralnetwork/RL/pytorch/non_parallel/DDPG.py new file mode 100644 index 00000000..4b0abd94 --- /dev/null +++ b/7.0/neuralnetwork/RL/pytorch/non_parallel/DDPG.py @@ -0,0 +1,101 @@ +import torch +import torch.nn.functional as F +import numpy as np +import gym + + +class actor(torch.nn.Module): + def __init__(self,state_dim,hidden_dim,action_dim,action_bound): + super(actor,self).__init__() + self.fc1=torch.nn.Linear(state_dim,hidden_dim) + self.fc2=torch.nn.Linear(hidden_dim,action_dim) + self.action_bound=action_bound + + + def forward(self,x): + x=F.relu(self.fc1(x)) + return torch.tanh(self.fc2(x))*self.action_bound + + +class critic(torch.nn.Module): + def __init__(self,state_dim,hidden_dim,action_dim): + super(critic,self).__init__() + self.fc1=torch.nn.Linear(state_dim+action_dim,hidden_dim) + self.fc2=torch.nn.Linear(hidden_dim,1) + + + def forward(self,x,a): + cat=torch.cat([x,a],dim=1) + x=F.relu(self.fc1(cat)) + return self.fc2(x) + + +class DDPG: + def __init__(self,hidden_dim,sigma,gamma,tau,actor_lr,critic_lr): + if torch.cuda.is_available(): + self.device=torch.device('cuda') + else: + self.device=torch.device('cpu') + self.genv=gym.make('Pendulum-v0') + state_dim=self.genv.observation_space.shape[0] + action_dim=self.genv.action_space.shape[0] + action_bound=self.genv.action_space.high[0] + self.actor=actor(state_dim,hidden_dim,action_dim,action_bound).to(self.device) + self.critic=critic(state_dim,hidden_dim,action_dim).to(self.device) + self.target_actor=actor(state_dim,hidden_dim,action_dim,action_bound).to(self.device) + self.target_critic=critic(state_dim,hidden_dim,action_dim).to(self.device) + self.target_actor.load_state_dict(self.actor.state_dict()) + self.target_critic.load_state_dict(self.critic.state_dict()) + self.sigma=sigma + self.gamma=gamma + self.tau=tau + self.actor_opt=torch.optim.Adam(self.actor.parameters(),lr=actor_lr) + self.critic_opt=torch.optim.Adam(self.critic.parameters(),lr=critic_lr) + + + def noise(self): + return np.random.normal(scale=self.sigma) + + + def env(self,a=None,initial=None): + if initial==True: + state=self.genv.reset(seed=0) + return state + else: + next_state,reward,done,_=self.genv.step(a) + return next_state,reward,done + + + def loss(self,s,a,next_s,r,d): + s=torch.tensor(s,dtype=torch.float).to(self.device) + a=torch.tensor(a,dtype=torch.float).view(-1,1).to(self.device) + next_s=torch.tensor(next_s,dtype=torch.float).to(self.device) + r=torch.tensor(r,dtype=torch.float).view(-1,1).to(self.device) + d=torch.tensor(d,dtype=torch.float).view(-1,1).to(self.device) + next_q_value=self.target_critic(next_s,self.target_actor(next_s)) + q_target=r+self.gamma*next_q_value*(1-d) + actor_loss=-torch.mean(self.critic(s,self.actor(s))) + critic_loss=F.mse_loss(self.critic(s,a),q_target) + return [actor_loss,critic_loss] + + + def backward(self,loss): + self.actor_opt.zero_grad() + loss[0].backward() + self.critic_opt.zero_grad() + loss[1].backward() + return + + + def opt(self): + self.actor_opt.step() + self.critic_opt.step() + return + + + def update_param(self): + for target_param,param in zip(self.target_actor.parameters(),self.actor.parameters()): + target_param.data.copy_(target_param.data*(1.0-self.tau)+param.data*self.tau) + for target_param,param in zip(self.target_critic.parameters(),self.critic.parameters()): + target_param.data.copy_(target_param.data*(1.0-self.tau)+param.data*self.tau) + return \ No newline at end of file diff --git a/7.0/neuralnetwork/RL/pytorch/non_parallel/DQN.py b/7.0/neuralnetwork/RL/pytorch/non_parallel/DQN.py new file mode 100644 index 00000000..a0e58689 --- /dev/null +++ b/7.0/neuralnetwork/RL/pytorch/non_parallel/DQN.py @@ -0,0 +1,64 @@ +import torch +import gym +import torch.nn.functional as F + + +class Qnet(torch.nn.Module): + def __init__(self,state_dim,hidden_dim,action_dim): + super(Qnet,self).__init__() + self.fc1=torch.nn.Linear(state_dim,hidden_dim) + self.fc2=torch.nn.Linear(hidden_dim,action_dim) + + + def forward(self,x): + x=F.relu(self.fc1(x)) + return self.fc2(x) + + +class DQN: + def __init__(self,state_dim,hidden_dim,action_dim): + if torch.cuda.is_available(): + self.device=torch.device('cuda') + else: + self.device=torch.device('cpu') + self.nn=Qnet(state_dim,hidden_dim,action_dim).to(self.device) + self.target_q_net=Qnet(state_dim,hidden_dim,action_dim).to(self.device) + self.optimizer=torch.optim.Adam(self.nn.parameters(),lr=2e-3) #optimizer,kernel uses it to optimize. + self.genv=gym.make('CartPole-v0') #create environment + + + def env(self,a=None,initial=None): #environment function,kernel uses it to interact with the environment. + if initial==True: + state=self.genv.reset() + return state + else: + next_state,reward,done,_=self.genv.step(a) + return next_state,reward,done + + + def loss(self,s,a,next_s,r,d): #loss function,kernel uses it to calculate loss. + s=torch.tensor(s,dtype=torch.float).to(self.device) + a=torch.tensor(a,dtype=torch.int64).view(-1,1).to(self.device) + next_s=torch.tensor(next_s,dtype=torch.float).to(self.device) + r=torch.tensor(r,dtype=torch.float).view(-1,1).to(self.device) + d=torch.tensor(d,dtype=torch.float).view(-1,1).to(self.device) + q_value=self.nn(s).gather(1,a) + next_q_value=self.target_q_net(next_s).max(1)[0].view(-1,1) + target=r+0.98*next_q_value*(1-d) + return F.mse_loss(q_value,target) + + + def backward(self,loss): #backward function,kernel uses it for backpropagation. + self.optimizer.zero_grad() + loss.backward() + return + + + def opt(self): #opt function,kernel uses it to optimize. + self.optimizer.step() + return + + + def update_param(self): #update function,kernel uses it to update parameter. + self.target_q_net.load_state_dict(self.nn.state_dict()) + return \ No newline at end of file diff --git a/7.0/neuralnetwork/RL/pytorch/non_parallel/DQN_pr.py b/7.0/neuralnetwork/RL/pytorch/non_parallel/DQN_pr.py new file mode 100644 index 00000000..66c71847 --- /dev/null +++ b/7.0/neuralnetwork/RL/pytorch/non_parallel/DQN_pr.py @@ -0,0 +1,76 @@ +import torch +import gym +import torch.nn.functional as F +import Note.create.RL.rl.prioritized_replay as pr + +#prioritized replay example +class Qnet(torch.nn.Module): + def __init__(self,state_dim,hidden_dim,action_dim): + super(Qnet,self).__init__() + self.fc1=torch.nn.Linear(state_dim,hidden_dim) + self.fc2=torch.nn.Linear(hidden_dim,action_dim) + + + def forward(self,x): + x=F.relu(self.fc1(x)) + return self.fc2(x) + + +class DQN: + def __init__(self,state_dim,hidden_dim,action_dim): + if torch.cuda.is_available(): + self.device=torch.device('cuda') + else: + self.device=torch.device('cpu') + self.nn=Qnet(state_dim,hidden_dim,action_dim).to(self.device) + self.target_q_net=Qnet(state_dim,hidden_dim,action_dim).to(self.device) + self.pr=pr.pr() + self.initial_TD=7 + self._epsilon=0.0007 + self.alpha=0.7 + self.optimizer=torch.optim.Adam(self.nn.parameters(),lr=2e-3) + self.genv=gym.make('CartPole-v0') + + + def env(self,a=None,initial=None): + if initial==True: + state=self.genv.reset(seed=0) + return state + else: + next_state,reward,done,_=self.genv.step(a) + return next_state,reward,done + + + def data_func(self,state_pool,action_pool,next_state_pool,reward_pool,done_pool,batch): + s,a,next_s,r,d=self.pr.sample(state_pool,action_pool,next_state_pool,reward_pool,done_pool,self.epsilon,self.alpha,batch) + return s,a,next_s,r,d + + + def loss(self,s,a,next_s,r,d): + s=torch.tensor(s,dtype=torch.float).to(self.device) + a=torch.tensor(a,dtype=torch.int64).view(-1,1).to(self.device) + next_s=torch.tensor(next_s,dtype=torch.float).to(self.device) + r=torch.tensor(r,dtype=torch.float).view(-1,1).to(self.device) + d=torch.tensor(d,dtype=torch.float).view(-1,1).to(self.device) + q_value=self.nn(s).gather(1,a) + next_q_value=self.target_q_net(next_s).max(1)[0].view(-1,1) + target=r+0.98*next_q_value*(1-d) + TD=target-q_value + self.pr.update_TD(TD) + return torch.mean(TD**2) + + + def backward(self,loss): + self.optimizer.zero_grad() + loss.backward() + return + + + def opt(self): + self.optimizer.step() + return + + + def update_param(self): + self.target_q_net.load_state_dict(self.nn.state_dict()) + return \ No newline at end of file diff --git a/7.0/neuralnetwork/RL/pytorch/non_parallel/DoubleDQN.py b/7.0/neuralnetwork/RL/pytorch/non_parallel/DoubleDQN.py new file mode 100644 index 00000000..31e606e8 --- /dev/null +++ b/7.0/neuralnetwork/RL/pytorch/non_parallel/DoubleDQN.py @@ -0,0 +1,65 @@ +import torch +import gym +import torch.nn.functional as F + + +class Qnet(torch.nn.Module): + def __init__(self,state_dim,hidden_dim,action_dim): + super(Qnet,self).__init__() + self.fc1=torch.nn.Linear(state_dim,hidden_dim) + self.fc2=torch.nn.Linear(hidden_dim,action_dim) + + + def forward(self,x): + x=F.relu(self.fc1(x)) + return self.fc2(x) + + +class DoubleDQN: + def __init__(self,state_dim,hidden_dim,action_dim): + if torch.cuda.is_available(): + self.device=torch.device('cuda') + else: + self.device=torch.device('cpu') + self.nn=Qnet(state_dim,hidden_dim,action_dim).to(self.device) + self.target_q_net=Qnet(state_dim,hidden_dim,action_dim).to(self.device) + self.optimizer=torch.optim.Adam(self.nn.parameters(),lr=2e-3) + self.genv=gym.make('CartPole-v0') + + + def env(self,a=None,initial=None): + if initial==True: + state=self.genv.reset(seed=0) + return state + else: + next_state,reward,done,_=self.genv.step(a) + return next_state,reward,done + + + def loss(self,s,a,next_s,r,d): + s=torch.tensor(s,dtype=torch.float).to(self.device) + a=torch.tensor(a,dtype=torch.int64).view(-1,1).to(self.device) + next_s=torch.tensor(next_s,dtype=torch.float).to(self.device) + r=torch.tensor(r,dtype=torch.float).view(-1,1).to(self.device) + d=torch.tensor(d,dtype=torch.float).view(-1,1).to(self.device) + q_value=self.nn(s).gather(1,a) + max_action=self.nn(next_s).max(1)[1].view(-1,1) + next_q_value=self.target_q_net(next_s).gather(1,max_action) + target=r+0.98*next_q_value*(1-d) + return F.mse_loss(q_value,target) + + + def backward(self,loss): + self.optimizer.zero_grad() + loss.backward() + return + + + def opt(self): + self.optimizer.step() + return + + + def update_param(self): + self.target_q_net.load_state_dict(self.nn.state_dict()) + return \ No newline at end of file diff --git a/7.0/neuralnetwork/RL/pytorch/non_parallel/DuelingDQN.py b/7.0/neuralnetwork/RL/pytorch/non_parallel/DuelingDQN.py new file mode 100644 index 00000000..b81eeb37 --- /dev/null +++ b/7.0/neuralnetwork/RL/pytorch/non_parallel/DuelingDQN.py @@ -0,0 +1,67 @@ +import torch +import gym +import torch.nn.functional as F + + +class VAnet(torch.nn.Module): + def __init__(self,state_dim,hidden_dim,action_dim): + super(VAnet,self).__init__() + self.fc1=torch.nn.Linear(state_dim,hidden_dim) + self.fc_A=torch.nn.Linear(hidden_dim,action_dim) + self.fc_V=torch.nn.Linear(hidden_dim,1) + + + def forward(self,x): + A=self.fc_A(F.relu(self.fc1(x))) + V=self.fc_V(F.relu(self.fc1(x))) + Q=V+A-A.mean(1).view(-1,1) + return Q + + +class DuelingDQN: + def __init__(self,state_dim,hidden_dim,action_dim): + if torch.cuda.is_available(): + self.device=torch.device('cuda') + else: + self.device=torch.device('cpu') + self.nn=VAnet(state_dim,hidden_dim,action_dim).to(self.device) + self.target_q_net=VAnet(state_dim,hidden_dim,action_dim).to(self.device) + self.optimizer=torch.optim.Adam(self.nn.parameters(),lr=2e-3) + self.genv=gym.make('CartPole-v0') + + + def env(self,a=None,initial=None): + if initial==True: + state=self.genv.reset(seed=0) + return state + else: + next_state,reward,done,_=self.genv.step(a) + return next_state,reward,done + + + def loss(self,s,a,next_s,r,d): + s=torch.tensor(s,dtype=torch.float).to(self.device) + a=torch.tensor(a,dtype=torch.int64).view(-1,1).to(self.device) + next_s=torch.tensor(next_s,dtype=torch.float).to(self.device) + r=torch.tensor(r,dtype=torch.float).view(-1,1).to(self.device) + d=torch.tensor(d,dtype=torch.float).view(-1,1).to(self.device) + q_value=self.nn(s).gather(1,a) + next_q_value=self.target_q_net(next_s).max(1)[0].view(-1,1) + target=r+0.98*next_q_value*(1-d) + return F.mse_loss(q_value,target) + + + def backward(self,loss): + self.optimizer.zero_grad() + loss.backward() + return + + + def opt(self): + self.optimizer.step() + return + + + def update_param(self): + self.target_q_net.load_state_dict(self.nn.state_dict()) + return \ No newline at end of file diff --git a/7.0/neuralnetwork/RL/pytorch/parallel/DQN.py b/7.0/neuralnetwork/RL/pytorch/parallel/DQN.py new file mode 100644 index 00000000..f4d8417d --- /dev/null +++ b/7.0/neuralnetwork/RL/pytorch/parallel/DQN.py @@ -0,0 +1,65 @@ +import torch +import gym +import torch.nn.functional as F +from Note.nn.parallel.assign_device_pytorch import assign_device + + +class Qnet(torch.nn.Module): + def __init__(self,state_dim,hidden_dim,action_dim): + super(Qnet,self).__init__() + self.fc1=torch.nn.Linear(state_dim,hidden_dim) + self.fc2=torch.nn.Linear(hidden_dim,action_dim) + + + def forward(self,x): + x=F.relu(self.fc1(x)) + return self.fc2(x) + + +class DQN: + def __init__(self,state_dim,hidden_dim,action_dim): + if torch.cuda.is_available(): + self.device=torch.device('cuda') + else: + self.device=torch.device('cpu') + self.nn=Qnet(state_dim,hidden_dim,action_dim).to(self.device) + self.target_q_net=Qnet(state_dim,hidden_dim,action_dim).to(self.device) + self.optimizer=[torch.optim.Adam(self.nn.parameters(),lr=2e-3) for _ in range(5)] #optimizer,kernel uses it to optimize. + self.genv=[gym.make('CartPole-v0') for _ in range(5)] #create environment + + + def env(self,a=None,p=None,initial=None): #environment function,kernel uses it to interact with the environment. + if initial==True: + state=self.genv[p].reset() + return state + else: + next_state,reward,done,_=self.genv[p].step(a) + return next_state,reward,done + + + def loss(self,s,a,next_s,r,d,p): #loss function,kernel uses it to calculate loss. + s=torch.tensor(s,dtype=torch.float).to(assign_device(p,'GPU')) + a=torch.tensor(a,dtype=torch.int64).view(-1,1).to(assign_device(p,'GPU')) + next_s=torch.tensor(next_s,dtype=torch.float).to(assign_device(p,'GPU')) + r=torch.tensor(r,dtype=torch.float).view(-1,1).to(assign_device(p,'GPU')) + d=torch.tensor(d,dtype=torch.float).view(-1,1).to(assign_device(p,'GPU')) + q_value=self.nn(s).gather(1,a) + next_q_value=self.target_q_net(next_s).max(1)[0].view(-1,1) + target=r+0.98*next_q_value*(1-d) + return F.mse_loss(q_value,target) + + + def backward(self,loss,p): #backward function,kernel uses it for backpropagation. + self.optimizer[p].zero_grad(set_to_none=True) + loss.backward() + return + + + def opt(self,p): #opt function,kernel uses it to optimize. + self.optimizer[p].step() + return + + + def update_param(self): #update function,kernel uses it to update parameter. + self.target_q_net.load_state_dict(self.nn.state_dict()) + return \ No newline at end of file diff --git a/7.0/neuralnetwork/RL/tensorflow/non_parallel/DDPG.py b/7.0/neuralnetwork/RL/tensorflow/non_parallel/DDPG.py new file mode 100644 index 00000000..d3666ee1 --- /dev/null +++ b/7.0/neuralnetwork/RL/tensorflow/non_parallel/DDPG.py @@ -0,0 +1,84 @@ +import tensorflow as tf # import TensorFlow library +import numpy as np # import NumPy library +import gym # import OpenAI Gym library + + +class actor: # define a class for the actor network + def __init__(self,state_dim,hidden_dim,action_dim,action_bound): # initialize the network with state dimension, hidden dimension, action dimension and action bound + self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) # create a weight matrix for the first layer + self.bias1=tf.Variable(tf.random.normal([hidden_dim])) # create a bias vector for the first layer + self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) # create a weight matrix for the second layer + self.bias2=tf.Variable(tf.random.normal([action_dim])) # create a bias vector for the second layer + self.action_bound=action_bound # store the action bound + self.param=[self.weight1,self.bias1,self.weight2,self.bias2] # store the network parameters in a list + + + def fp(self,x): # forward propagation function, kernel uses it for forward propagation + x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) # apply the first layer with ReLU activation + return tf.nn.tanh(tf.matmul(x,self.weight2)+self.bias2)*self.action_bound # apply the second layer with tanh activation and scale it by the action bound + + +class critic: # define a class for the critic network + def __init__(self,state_dim,hidden_dim,action_dim): # initialize the network with state dimension, hidden dimension and action dimension + self.weight1=tf.Variable(tf.random.normal([state_dim+action_dim,hidden_dim])) # create a weight matrix for the first layer + self.bias1=tf.Variable(tf.random.normal([hidden_dim])) # create a bias vector for the first layer + self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) # create a weight matrix for the second layer + self.bias2=tf.Variable(tf.random.normal([action_dim])) # create a bias vector for the second layer + self.param=[self.weight1,self.bias1,self.weight2,self.bias2] # store the network parameters in a list + + + def fp(self,x,a): # forward propagation function, kernel uses it for forward propagation + cat=tf.concat([x,a],axis=1) # concatenate the state and action vectors along the first axis + x=tf.nn.relu(tf.matmul(cat,self.weight1)+self.bias1) # apply the first layer with ReLU activation + return tf.matmul(x,self.weight2)+self.bias2 # apply the second layer and return the output + + +class DDPG: # define a class for the DDPG agent + def __init__(self,hidden_dim,sigma,gamma,tau,actor_lr,critic_lr): # initialize the agent with hidden dimension, noise scale, discount factor, soft update factor, actor learning rate and critic learning rate + self.genv=gym.make('Pendulum-v0') # create environment + state_dim=self.genv.observation_space.shape[0] # get the state dimension from the environment observation space + action_dim=self.genv.action_space.shape[0] # get the action dimension from the environment action space + action_bound=self.genv.action_space.high[0] # get the action bound from the environment action space high value (assume symmetric) + self.actor=actor(state_dim,hidden_dim,action_dim,action_bound) # create an actor network for the agent + self.critic=critic(state_dim,hidden_dim,action_dim) # create a critic network for the agent + self.target_actor=actor(state_dim,hidden_dim,action_dim,action_bound) # create a target actor network for the agent + self.target_critic=critic(state_dim,hidden_dim,action_dim) # create a target critic network for the agent + self.actor_param=self.actor.param # parameter list of actor network, kernel uses it list for backpropagation + self.critic_param=self.critic.param # parameter list of critic network, kernel uses it list for backpropagation + self.target_actor.param=self.actor_param.copy() # copy the parameters from actor network to target actor network + self.target_critic.param=self.critic_param.copy() # copy the parameters from critic network to target critic network + self.param=[self.actor_param,self.critic_param] # parameter list of both networks, kernel uses it list for backpropagation + self.sigma=sigma # noise scale + self.gamma=gamma # discount factor + self.tau=tau # soft update factor + self.opt=tf.keras.optimizers.Adam() # optimizer, kernel uses it to optimize. Here we use Adam optimizer + + + def noise(self): # noise function, kernel uses it to generate exploration noise + return np.random.normal(scale=self.sigma) # return a random sample from a normal distribution with zero mean and sigma scale + + + def env(self,a=None,initial=None): # environment function, kernel uses it to interact with the environment + if initial==True: # if initial is True, reset the environment and return the initial state + state=self.genv.reset(seed=0) + return state + else: # otherwise, take an action and return the next state, reward and done flag + next_state,reward,done,_=self.genv.step(a) + return next_state,reward,done + + + def loss(self,s,a,next_s,r,d): # loss function, kernel uses it to calculate loss + a=tf.expand_dims(a,axis=1) # expand the action vector to match the Q-value matrix shape + next_q_value=self.target_critic.fp(next_s,self.target_actor.fp(next_s)) # get the Q-value for the next state and action from the target critic network + q_target=r+self.gamma*next_q_value*(1-d) # calculate the target value using Bellman equation with discount factor gamma + actor_loss=-tf.reduce_mean(self.critic.fp(s,self.actor.fp(s))) # calculate the actor loss as the negative mean of the Q-value for the current state and action from the critic network + critic_loss=tf.reduce_mean((self.critic.fp(s,a)-q_target)**2) # calculate the critic loss as the mean squared error between Q-value and target value + return [actor_loss,critic_loss] # return a list of actor loss and critic loss + + + def update_param(self): # update function, kernel uses it to update parameter + for target_param,param in zip(self.target_actor.param,self.actor.param): # for each pair of parameters in target actor network and actor network + target_param=target_param*(1.0-self.tau)+param*self.tau # update the target parameter using soft update with factor tau + for target_param,param in zip(self.target_critic.param,self.critic.param): # for each pair of parameters in target critic network and critic network + target_param=target_param*(1.0-self.tau)+param*self.tau # update the target parameter using soft update with factor tau + return \ No newline at end of file diff --git a/7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN.py b/7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN.py new file mode 100644 index 00000000..21487674 --- /dev/null +++ b/7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN.py @@ -0,0 +1,46 @@ +import tensorflow as tf # import TensorFlow library +import gym # import OpenAI Gym library + + +class Qnet: # define a class for the Q-network + def __init__(self,state_dim,hidden_dim,action_dim): # initialize the network with state dimension, hidden dimension and action dimension + self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) # create a weight matrix for the first layer + self.bias1=tf.Variable(tf.random.normal([hidden_dim])) # create a bias vector for the first layer + self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) # create a weight matrix for the second layer + self.bias2=tf.Variable(tf.random.normal([action_dim])) # create a bias vector for the second layer + self.param=[self.weight1,self.bias1,self.weight2,self.bias2] # store the network parameters in a list + + def fp(self,x): # forward propagation function, kernel uses it for forward propagation + x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) # apply the first layer with ReLU activation + return tf.matmul(x,self.weight2)+self.bias2 # apply the second layer and return the output + + +class DQN: # define a class for the DQN agent + def __init__(self,state_dim,hidden_dim,action_dim): # initialize the agent with state dimension, hidden dimension and action dimension + self.nn=Qnet(state_dim,hidden_dim,action_dim) # create a Q-network for the agent + self.target_q_net=Qnet(state_dim,hidden_dim,action_dim) # create a target Q-network for the agent + self.param=self.nn.param # parameter list, kernel uses it list for backpropagation + self.opt=tf.keras.optimizers.Adam() # optimizer, kernel uses it to optimize + self.genv=gym.make('CartPole-v0') # create environment + + + def env(self,a=None,initial=None): # environment function, kernel uses it to interact with the environment + if initial==True: # if initial is True, reset the environment and return the initial state + state=self.genv.reset(seed=0) + return state + else: # otherwise, take an action and return the next state, reward and done flag + next_state,reward,done,_=self.genv.step(a) + return next_state,reward,done + + + def loss(self,s,a,next_s,r,d): # loss function, kernel uses it to calculate loss + a=tf.expand_dims(a,axis=1) # expand the action vector to match the Q-value matrix shape + q_value=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) # get the Q-value for the selected action + next_q_value=tf.reduce_max(self.target_q_net.fp(next_s),axis=1) # get the maximum Q-value for the next state from the target network + target=r+0.98*next_q_value*(1-d) # calculate the target value using Bellman equation with discount factor 0.98 + return tf.reduce_mean((q_value-target)**2) # return the mean squared error between Q-value and target value + + + def update_param(self): # update function, kernel uses it to update parameter + self.target_q_net.param=self.param.copy() # copy the parameters from the Q-network to the target network + return \ No newline at end of file diff --git a/7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN_pr.py b/7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN_pr.py new file mode 100644 index 00000000..77e8b8a9 --- /dev/null +++ b/7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN_pr.py @@ -0,0 +1,59 @@ +import tensorflow as tf +import gym +import Note.create.RL.rl.prioritized_replay as pr + + +class Qnet: + def __init__(self,state_dim,hidden_dim,action_dim): + self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) + self.bias1=tf.Variable(tf.random.normal([hidden_dim])) + self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) + self.bias2=tf.Variable(tf.random.normal([action_dim])) + self.param=[self.weight1,self.bias1,self.weight2,self.bias2] + + def fp(self,x): + x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) + return tf.matmul(x,self.weight2)+self.bias2 + + +class DQN: + def __init__(self,state_dim,hidden_dim,action_dim): + self.nn=Qnet(state_dim,hidden_dim,action_dim) + self.target_q_net=Qnet(state_dim,hidden_dim,action_dim) + self.param=self.nn.param + self.pr=pr.pr() + self.initial_TD=7 + self._epsilon=0.0007 + self.alpha=0.7 + self.opt=tf.keras.optimizers.Adam() + self.genv=gym.make('CartPole-v0') + + + def env(self,a=None,initial=None): + if initial==True: + state=self.genv.reset(seed=0) + return state + else: + next_state,reward,done,_=self.genv.step(a) + return next_state,reward,done + + + def data_func(self,state_pool,action_pool,next_state_pool,reward_pool,done_pool,batch): + s,a,next_s,r,d=self.pr.sample(state_pool,action_pool,next_state_pool,reward_pool,done_pool,self.epsilon,self.alpha,batch) + return s,a,next_s,r,d + + + def loss(self,s,a,next_s,r,d): + a=tf.expand_dims(a,axis=1) + q_value=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) + next_q_value=tf.reduce_max(self.target_q_net.fp(next_s),axis=1) + target=r+0.98*next_q_value*(1-d) + target=tf.expand_dims(target,axis=1) + TD=target-q_value + self.pr.update_TD(TD) + return tf.reduce_mean(TD**2) + + + def update_param(self): + self.target_q_net.param=self.param.copy() + return \ No newline at end of file diff --git a/7.0/neuralnetwork/RL/tensorflow/parallel/DQN.py b/7.0/neuralnetwork/RL/tensorflow/parallel/DQN.py new file mode 100644 index 00000000..d7756475 --- /dev/null +++ b/7.0/neuralnetwork/RL/tensorflow/parallel/DQN.py @@ -0,0 +1,52 @@ +import tensorflow as tf # import TensorFlow library +import gym # import OpenAI Gym library +import Note.nn.parallel.optimizer as o # import Note's optimizer module + + +class Qnet: # define a class for the Q-network + def __init__(self,state_dim,hidden_dim,action_dim): # initialize the network with state dimension, hidden dimension and action dimension + self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) # create a weight matrix for the first layer + self.bias1=tf.Variable(tf.random.normal([hidden_dim])) # create a bias vector for the first layer + self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) # create a weight matrix for the second layer + self.bias2=tf.Variable(tf.random.normal([action_dim])) # create a bias vector for the second layer + self.param=[self.weight1,self.bias1,self.weight2,self.bias2] # store the network parameters in a list + + def fp(self,x): # forward propagation function, kernel uses it for forward propagation + x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) # apply the first layer with ReLU activation + return tf.matmul(x,self.weight2)+self.bias2 # apply the second layer and return the output + + +class DQN: # define a class for the DQN agent + def __init__(self,state_dim,hidden_dim,action_dim): # initialize the agent with state dimension, hidden dimension and action dimension + self.nn=Qnet(state_dim,hidden_dim,action_dim) # create a Q-network for the agent + self.target_q_net=Qnet(state_dim,hidden_dim,action_dim) # create a target Q-network for the agent + self.param=self.nn.param # parameter list, kernel uses it list for backpropagation + self.optimizer=o.Momentum(0.07,0.7) # optimizer, kernel uses it to optimize. Here we use a custom momentum optimizer with learning rate 0.07 and momentum 0.7 + self.genv=[gym.make('CartPole-v0') for _ in range(5)] # create a list of 5 environments + + + def env(self,a=None,p=None,initial=None): # environment function, kernel uses it to interact with the environment + if initial==True: # if initial is True, reset the environment with index p and return the initial state + state=self.genv[p].reset(seed=0) + return state + else: # otherwise, take an action and return the next state, reward and done flag from the environment with index p + next_state,reward,done,_=self.genv[p].step(a) + return next_state,reward,done + + + def loss(self,s,a,next_s,r,d): # loss function, kernel uses it to calculate loss + a=tf.expand_dims(a,axis=1) # expand the action vector to match the Q-value matrix shape + q_value=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) # get the Q-value for the selected action + next_q_value=tf.reduce_max(self.target_q_net.fp(next_s),axis=1) # get the maximum Q-value for the next state from the target network + target=r+0.98*next_q_value*(1-d) # calculate the target value using Bellman equation with discount factor 0.98 + return tf.reduce_mean((q_value-target)**2) # return the mean squared error between Q-value and target value + + + def update_param(self): # update function, kernel uses it to update parameter + self.target_q_net.param=self.param.copy() # copy the parameters from the Q-network to the target network + return + + + def opt(self,gradient): # optimization function, kernel uses it to optimize parameter + param=self.optimizer.opt(gradient,self.param) # apply the custom momentum optimizer to update the parameters using the gradient + return param \ No newline at end of file From c933834248e2d7d8463c62f08342548ccad80457 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 24 Aug 2023 14:54:02 +0800 Subject: [PATCH 178/337] Update transformer.py --- 7.0/neuralnetwork/DL/tensorflow/parallel/transformer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/transformer.py b/7.0/neuralnetwork/DL/tensorflow/parallel/transformer.py index ae052538..d367c6ad 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/transformer.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/transformer.py @@ -28,7 +28,7 @@ def build(self): # Create a linear layer to map the transformer output to logits self.logits_layer = dense([self.embed_size,self.vocab_size]) # Store the parameters of the layers in a list - self.param = [self.embedding_layer.embeddings] + self.param = [self.embedding_layer.param] for transformer_layer in self.transformer_layers: self.param.extend(transformer_layer.param) self.param.extend(self.logits_layer.param) @@ -52,4 +52,4 @@ def loss(self,output,labels): def opt(self,gradient): # Perform optimization on the parameters using the gradient and the batch count param=self.optimizer.opt(gradient,self.param,self.bc[0]) - return param \ No newline at end of file + return param From efb18105cb9c2b5291137d4f2510a93a7288f746 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 24 Aug 2023 14:54:15 +0800 Subject: [PATCH 179/337] Update self_LSTM.py --- 7.0/neuralnetwork/DL/tensorflow/parallel/self_LSTM.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/self_LSTM.py b/7.0/neuralnetwork/DL/tensorflow/parallel/self_LSTM.py index 85af312a..72fd6687 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/self_LSTM.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/self_LSTM.py @@ -28,7 +28,7 @@ def build(self): # Create a linear layer to map the attention output to logits self.logits_layer = dense([self.embed_size,self.vocab_size]) # Store the parameters of the layers in a list - self.param = [self.embedding_layer.embeddings] + self.param = [self.embedding_layer.param] self.param.extend(self.lstm_layer.param) self.param.extend(self.self_attention_layer.param) self.param.extend(self.logits_layer.param) @@ -51,4 +51,4 @@ def loss(self,output,labels): def opt(self,gradient): # Perform optimization on the parameters using the gradient and the batch count param=self.optimizer.opt(gradient,self.param,self.bc[0]) - return param \ No newline at end of file + return param From 3015f171044d71bd662afc43126a39d1a946c5e5 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 24 Aug 2023 14:54:26 +0800 Subject: [PATCH 180/337] Update cnn_lmix.py --- .../DL/tensorflow/parallel/cnn_lmix.py | 27 +++++++++---------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/cnn_lmix.py b/7.0/neuralnetwork/DL/tensorflow/parallel/cnn_lmix.py index b6e66458..5776dde6 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/cnn_lmix.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/cnn_lmix.py @@ -19,20 +19,19 @@ def __init__(self): def build(self): # Create three convolutional layers with relu activations - self.conv1=conv2d(weight_shape=(3,3,3,32),activation='relu') - self.conv2=conv2d(weight_shape=(3,3,32,64),activation='relu') - self.conv3=conv2d(weight_shape=(3,3,64,64),activation='relu') + self.conv1=conv2d(weight_shape=(3,3,3,32),strides=(1,1),padding='SAME',activation='relu') + self.conv2=conv2d(weight_shape=(3,3,32,64),strides=(1,1),padding='SAME',activation='relu') + self.conv3=conv2d(weight_shape=(3,3,64,64),strides=(1,1),padding='SAME',activation='relu') # Create two dense layers with relu and linear activations self.dense1=dense([64*4*4,64],activation='relu') self.dense2=dense([64,10]) # Store the parameters of the layers in a list - self.param = [self.conv1.weight, - self.conv2.weight, - self.conv3.weight, - self.dense1.weight, - self.dense1.bias, - self.dense2.weight, - self.dense2.bias] + self.param = [self.conv1.param, + self.conv2.param, + self.conv3.param, + self.dense1.param, + self.dense2.param, + ] return @@ -43,11 +42,11 @@ def data_func(self,data_batch,labels_batch): def fp(self,data): # Perform forward propagation on the input data - x=tf.nn.conv2d(data,self.conv1.weight,strides=(1,1),padding='SAME') # First convolutional layer + x=self.conv1.output(data) # First convolutional layer x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # First max pooling layer - x=tf.nn.conv2d(x,self.conv2.weight,strides=(1,1),padding='SAME') # Second convolutional layer + x=self.conv2.output(x) # Second convolutional layer x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # Second max pooling layer - x=tf.nn.conv2d(x,self.conv3.weight,strides=(1,1),padding='SAME') # Third convolutional layer + x=self.conv3.output(x) # Third convolutional layer x=flatten(x) # Flatten the output to a vector x=tf.nn.relu(tf.matmul(x,self.dense1.weight)+self.dense1.bias) # First dense layer with relu activation x=tf.nn.dropout(x,rate=0.5) # Apply dropout to prevent overfitting @@ -64,4 +63,4 @@ def loss(self,output,labels): def opt(self,gradient): # Perform optimization on the parameters using the gradient and the batch count param=self.optimizer.opt(gradient,self.param,self.bc[0]) - return param \ No newline at end of file + return param From 5ca7e16e49598a11d951cd7a9a520bc5b969070d Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 24 Aug 2023 14:54:36 +0800 Subject: [PATCH 181/337] Update cnn.py --- .../DL/tensorflow/parallel/cnn.py | 29 +++++++++---------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/cnn.py b/7.0/neuralnetwork/DL/tensorflow/parallel/cnn.py index 59be2e63..f99003e0 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/cnn.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/cnn.py @@ -16,30 +16,29 @@ def __init__(self): def build(self): # Create three convolutional layers with relu activations - self.conv1=conv2d(weight_shape=(3,3,3,32),activation='relu') - self.conv2=conv2d(weight_shape=(3,3,32,64),activation='relu') - self.conv3=conv2d(weight_shape=(3,3,64,64),activation='relu') + self.conv1=conv2d(weight_shape=(3,3,3,32),strides=(1,1),padding='SAME',activation='relu') + self.conv2=conv2d(weight_shape=(3,3,32,64),strides=(2,2),padding='VALID',activation='relu') + self.conv3=conv2d(weight_shape=(3,3,64,64),strides=(1,1),padding='SAME',activation='relu') # Create two dense layers with relu and linear activations self.dense1=dense([64*4*4,64],activation='relu') self.dense2=dense([64,10]) # Store the parameters of the layers in a list - self.param = [self.conv1.weight, - self.conv2.weight, - self.conv3.weight, - self.dense1.weight, - self.dense1.bias, - self.dense2.weight, - self.dense2.bias] + self.param = [self.conv1.param, + self.conv2.param, + self.conv3.param, + self.dense1.param, + self.dense2.param, + ] return def fp(self,data): # Perform forward propagation on the input data - x=tf.nn.conv2d(data,self.conv1.weight,strides=(1,1),padding='SAME') # First convolutional layer - x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # First max pooling layer - x=tf.nn.conv2d(x,self.conv2.weight,strides=(1,1),padding='SAME') # Second convolutional layer + x=self.conv1.output(data) # First convolutional layer + x=tf.nn.max_pool2d(x,ksize=(2,2)) # First max pooling layer + x=self.conv2.output(x) # Second convolutional layer x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # Second max pooling layer - x=tf.nn.conv2d(x,self.conv3.weight,strides=(1,1),padding='SAME') # Third convolutional layer + x=self.conv3.output(x) # Third convolutional layer x=flatten(x) # Flatten the output to a vector x=tf.nn.relu(tf.matmul(x,self.dense1.weight)+self.dense1.bias) # First dense layer with relu activation x=tf.nn.dropout(x,rate=0.5) # Apply dropout to prevent overfitting @@ -56,4 +55,4 @@ def loss(self,output,labels): def opt(self,gradient): # Perform optimization on the parameters using the gradient and the batch count param=self.optimizer.opt(gradient,self.param,self.bc[0]) - return param \ No newline at end of file + return param From ecc3541b2243dd9bc5b92664ec4135db88f6c2b6 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 24 Aug 2023 14:57:43 +0800 Subject: [PATCH 182/337] Update self_LSTM.py --- 7.0/neuralnetwork/DL/tensorflow/layer/self_LSTM.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/7.0/neuralnetwork/DL/tensorflow/layer/self_LSTM.py b/7.0/neuralnetwork/DL/tensorflow/layer/self_LSTM.py index edaa2420..de64b987 100644 --- a/7.0/neuralnetwork/DL/tensorflow/layer/self_LSTM.py +++ b/7.0/neuralnetwork/DL/tensorflow/layer/self_LSTM.py @@ -25,7 +25,7 @@ def build(self): # Create a linear layer to map the attention output to logits self.logits_layer = dense([self.embed_size,self.vocab_size]) # Store the parameters of the layers in a list - self.param = [self.embedding_layer.embeddings] + self.param = [self.embedding_layer.param] self.param.extend(self.lstm_layer.param) self.param.extend(self.self_attention_layer.param) self.param.extend(self.logits_layer.param) @@ -42,4 +42,4 @@ def fp(self,data): def loss(self,output,labels): # Compute the loss between the output and the labels - return self.loss_object(labels,output) \ No newline at end of file + return self.loss_object(labels,output) From 8b11ec31cc7a8b590423484372c1abe80099266d Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 24 Aug 2023 14:57:54 +0800 Subject: [PATCH 183/337] Update transformer.py --- 7.0/neuralnetwork/DL/tensorflow/layer/transformer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/7.0/neuralnetwork/DL/tensorflow/layer/transformer.py b/7.0/neuralnetwork/DL/tensorflow/layer/transformer.py index 47862006..e431fc95 100644 --- a/7.0/neuralnetwork/DL/tensorflow/layer/transformer.py +++ b/7.0/neuralnetwork/DL/tensorflow/layer/transformer.py @@ -25,7 +25,7 @@ def build(self): # Create a linear layer to map the transformer output to logits self.logits_layer = dense([self.embed_size,self.vocab_size]) # Store the parameters of the layers in a list - self.param = [self.embedding_layer.embeddings] + self.param = [self.embedding_layer.param] for transformer_layer in self.transformer_layers: self.param.extend(transformer_layer.param) self.param.extend(self.logits_layer.param) @@ -43,4 +43,4 @@ def fp(self,data): def loss(self,output,labels): # Compute the loss between the output and the labels - return self.loss_object(labels,output) \ No newline at end of file + return self.loss_object(labels,output) From fe60fcb0feea3d7384ae1092b85a4770f8a77f91 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 24 Aug 2023 14:58:05 +0800 Subject: [PATCH 184/337] Update cnn_lmix.py --- .../DL/tensorflow/layer/cnn_lmix.py | 27 +++++++++---------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/7.0/neuralnetwork/DL/tensorflow/layer/cnn_lmix.py b/7.0/neuralnetwork/DL/tensorflow/layer/cnn_lmix.py index 40232919..ec7d3f9b 100644 --- a/7.0/neuralnetwork/DL/tensorflow/layer/cnn_lmix.py +++ b/7.0/neuralnetwork/DL/tensorflow/layer/cnn_lmix.py @@ -16,20 +16,19 @@ def __init__(self): def build(self): # Create three convolutional layers with relu activations - self.conv1=conv2d(weight_shape=(3,3,3,32),activation='relu') - self.conv2=conv2d(weight_shape=(3,3,32,64),activation='relu') - self.conv3=conv2d(weight_shape=(3,3,64,64),activation='relu') + self.conv1=conv2d(weight_shape=(3,3,3,32),strides=(1,1),padding='SAME',activation='relu') + self.conv2=conv2d(weight_shape=(3,3,32,64),strides=(1,1),padding='SAME',activation='relu') + self.conv3=conv2d(weight_shape=(3,3,64,64),strides=(1,1),padding='SAME',activation='relu') # Create two dense layers with relu and linear activations self.dense1=dense([64*4*4,64],activation='relu') self.dense2=dense([64,10]) # Store the parameters of the layers in a list - self.param = [self.conv1.weight, - self.conv2.weight, - self.conv3.weight, - self.dense1.weight, - self.dense1.bias, - self.dense2.weight, - self.dense2.bias] + self.param = [self.conv1.param, + self.conv2.param, + self.conv3.param, + self.dense1.param, + self.dense2.param, + ] return @@ -40,11 +39,11 @@ def data_func(self,data_batch,labels_batch): def fp(self,data): # Perform forward propagation on the input data - x=tf.nn.conv2d(data,self.conv1.weight,strides=(1,1),padding='SAME') # First convolutional layer + x=self.conv1.output(data) # First convolutional layer x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # First max pooling layer - x=tf.nn.conv2d(x,self.conv2.weight,strides=(1,1),padding='SAME') # Second convolutional layer + x=self.conv2.output(x) # Second convolutional layer x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # Second max pooling layer - x=tf.nn.conv2d(x,self.conv3.weight,strides=(1,1),padding='SAME') # Third convolutional layer + x=self.conv3.output(x) # Third convolutional layer x=flatten(x) # Flatten the output to a vector x=tf.nn.relu(tf.matmul(x,self.dense1.weight)+self.dense1.bias) # First dense layer with relu activation x=tf.nn.dropout(x,rate=0.5) # Apply dropout to prevent overfitting @@ -55,4 +54,4 @@ def fp(self,data): def loss(self,output,labels): # Compute the loss between the output and the labels - return self.loss_object(labels,output) \ No newline at end of file + return self.loss_object(labels,output) From 68e808e4890cb4f941945554259a7af95badc02b Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 24 Aug 2023 14:58:15 +0800 Subject: [PATCH 185/337] Update cnn.py --- 7.0/neuralnetwork/DL/tensorflow/layer/cnn.py | 27 ++++++++++---------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/7.0/neuralnetwork/DL/tensorflow/layer/cnn.py b/7.0/neuralnetwork/DL/tensorflow/layer/cnn.py index aa2667f8..fe5b8a92 100644 --- a/7.0/neuralnetwork/DL/tensorflow/layer/cnn.py +++ b/7.0/neuralnetwork/DL/tensorflow/layer/cnn.py @@ -13,30 +13,29 @@ def __init__(self): def build(self): # Create three convolutional layers with relu activations - self.conv1=conv2d(weight_shape=(3,3,3,32),activation='relu') - self.conv2=conv2d(weight_shape=(3,3,32,64),activation='relu') - self.conv3=conv2d(weight_shape=(3,3,64,64),activation='relu') + self.conv1=conv2d(weight_shape=(3,3,3,32),strides=(1,1),padding='SAME',activation='relu') + self.conv2=conv2d(weight_shape=(3,3,32,64),strides=(1,1),padding='SAME',activation='relu') + self.conv3=conv2d(weight_shape=(3,3,64,64),strides=(1,1),padding='SAME',activation='relu') # Create two dense layers with relu and linear activations self.dense1=dense([64*4*4,64],activation='relu') self.dense2=dense([64,10]) # Store the parameters of the layers in a list - self.param = [self.conv1.weight, - self.conv2.weight, - self.conv3.weight, - self.dense1.weight, - self.dense1.bias, - self.dense2.weight, - self.dense2.bias] + self.param = [self.conv1.param, + self.conv2.param, + self.conv3.param, + self.dense1.param, + self.dense2.param, + ] return def fp(self,data): # Perform forward propagation on the input data - x=tf.nn.conv2d(data,self.conv1.weight,strides=(1,1),padding='SAME') # First convolutional layer + x=self.conv1.output(data) # First convolutional layer x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # First max pooling layer - x=tf.nn.conv2d(x,self.conv2.weight,strides=(1,1),padding='SAME') # Second convolutional layer + x=self.conv2.output(x) # Second convolutional layer x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # Second max pooling layer - x=tf.nn.conv2d(x,self.conv3.weight,strides=(1,1),padding='SAME') # Third convolutional layer + x=self.conv3.output(x) # Third convolutional layer x=flatten(x) # Flatten the output to a vector x=tf.nn.relu(tf.matmul(x,self.dense1.weight)+self.dense1.bias) # First dense layer with relu activation x=tf.nn.dropout(x,rate=0.5) # Apply dropout to prevent overfitting @@ -47,4 +46,4 @@ def fp(self,data): def loss(self,output,labels): # Compute the loss between the output and the labels - return self.loss_object(labels,output) \ No newline at end of file + return self.loss_object(labels,output) From a0c1c407d15e80535cc1bff409e52fbef2a10572 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Tue, 29 Aug 2023 16:29:27 +0800 Subject: [PATCH 186/337] Update nn_clipping.py --- 7.0/neuralnetwork/DL/tensorflow/layer/nn_clipping.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/7.0/neuralnetwork/DL/tensorflow/layer/nn_clipping.py b/7.0/neuralnetwork/DL/tensorflow/layer/nn_clipping.py index b53a43da..02944c12 100644 --- a/7.0/neuralnetwork/DL/tensorflow/layer/nn_clipping.py +++ b/7.0/neuralnetwork/DL/tensorflow/layer/nn_clipping.py @@ -13,8 +13,8 @@ def __init__(self): def build(self): # Create two dense layers with relu and linear activations - self.layer1=d.dense([784,128],activation='relu') - self.layer2=d.dense([128,10]) + self.layer1=d.dense(128,784,activation='relu') + self.layer2=d.dense(10,128) # Store the parameters of the layers in a list self.param=[self.layer1.param,self.layer2.param] return @@ -37,4 +37,4 @@ def gradient(self,tape,loss): # Compute the gradients of the loss with respect to the parameters grads=tape.gradient(loss,self.param) # Clip the gradients by value to avoid exploding gradients - return gradient_clipping(grads,'value',1.0) \ No newline at end of file + return gradient_clipping(grads,'value',1.0) From c371071983e95a866a9f52a65ad4d78804330a03 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Tue, 29 Aug 2023 16:29:39 +0800 Subject: [PATCH 187/337] Update nn_acc.py --- 7.0/neuralnetwork/DL/tensorflow/layer/nn_acc.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/7.0/neuralnetwork/DL/tensorflow/layer/nn_acc.py b/7.0/neuralnetwork/DL/tensorflow/layer/nn_acc.py index cad20961..771ad6ee 100644 --- a/7.0/neuralnetwork/DL/tensorflow/layer/nn_acc.py +++ b/7.0/neuralnetwork/DL/tensorflow/layer/nn_acc.py @@ -11,8 +11,8 @@ def __init__(self): def build(self): - self.layer1=d.dense([784,128],activation='relu') - self.layer2=d.dense([128,10]) + self.layer1=d.dense(128,784,activation='relu') + self.layer2=d.dense(10,128) self.param=[self.layer1.param,self.layer2.param] return @@ -29,4 +29,4 @@ def loss(self,output,labels): def accuracy(self,output,labels): - return self.train_accuracy(labels,output) \ No newline at end of file + return self.train_accuracy(labels,output) From 3e8d149b613ef410bab06453cb69164c8fb3ba86 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Tue, 29 Aug 2023 16:29:49 +0800 Subject: [PATCH 188/337] Update nn.py --- 7.0/neuralnetwork/DL/tensorflow/layer/nn.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/7.0/neuralnetwork/DL/tensorflow/layer/nn.py b/7.0/neuralnetwork/DL/tensorflow/layer/nn.py index 77ed7454..978e83c1 100644 --- a/7.0/neuralnetwork/DL/tensorflow/layer/nn.py +++ b/7.0/neuralnetwork/DL/tensorflow/layer/nn.py @@ -10,8 +10,8 @@ def __init__(self): def build(self): - self.layer1=d.dense([784,128],activation='relu') - self.layer2=d.dense([128,10]) + self.layer1=d.dense(128,784,activation='relu') + self.layer2=d.dense(10,128) self.param=[self.layer1.param,self.layer2.param] return @@ -24,4 +24,4 @@ def fp(self,data): def loss(self,output,labels): - return self.loss_object(labels,output) \ No newline at end of file + return self.loss_object(labels,output) From ab00832b15ab195c9f25f49a7270cd38bb32f2d7 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Tue, 29 Aug 2023 16:30:00 +0800 Subject: [PATCH 189/337] Update cnn_lmix.py --- 7.0/neuralnetwork/DL/tensorflow/layer/cnn_lmix.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/7.0/neuralnetwork/DL/tensorflow/layer/cnn_lmix.py b/7.0/neuralnetwork/DL/tensorflow/layer/cnn_lmix.py index ec7d3f9b..76c97263 100644 --- a/7.0/neuralnetwork/DL/tensorflow/layer/cnn_lmix.py +++ b/7.0/neuralnetwork/DL/tensorflow/layer/cnn_lmix.py @@ -16,12 +16,12 @@ def __init__(self): def build(self): # Create three convolutional layers with relu activations - self.conv1=conv2d(weight_shape=(3,3,3,32),strides=(1,1),padding='SAME',activation='relu') - self.conv2=conv2d(weight_shape=(3,3,32,64),strides=(1,1),padding='SAME',activation='relu') - self.conv3=conv2d(weight_shape=(3,3,64,64),strides=(1,1),padding='SAME',activation='relu') + self.conv1=conv2d(32,[3,3],3,strides=(1,1),padding='SAME',activation='relu') + self.conv2=conv2d(64,[3,3],32,strides=(1,1),padding='SAME',activation='relu') + self.conv3=conv2d(64,[3,3],64,strides=(1,1),padding='SAME',activation='relu') # Create two dense layers with relu and linear activations - self.dense1=dense([64*4*4,64],activation='relu') - self.dense2=dense([64,10]) + self.dense1=dense(64,64*4*4,activation='relu') + self.dense2=dense(10,64) # Store the parameters of the layers in a list self.param = [self.conv1.param, self.conv2.param, From 8d49a222c40f4d619a0fe1ea5161d48d1980d55b Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Tue, 29 Aug 2023 16:30:12 +0800 Subject: [PATCH 190/337] Update cnn.py --- 7.0/neuralnetwork/DL/tensorflow/layer/cnn.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/7.0/neuralnetwork/DL/tensorflow/layer/cnn.py b/7.0/neuralnetwork/DL/tensorflow/layer/cnn.py index fe5b8a92..44ed6393 100644 --- a/7.0/neuralnetwork/DL/tensorflow/layer/cnn.py +++ b/7.0/neuralnetwork/DL/tensorflow/layer/cnn.py @@ -13,12 +13,12 @@ def __init__(self): def build(self): # Create three convolutional layers with relu activations - self.conv1=conv2d(weight_shape=(3,3,3,32),strides=(1,1),padding='SAME',activation='relu') - self.conv2=conv2d(weight_shape=(3,3,32,64),strides=(1,1),padding='SAME',activation='relu') - self.conv3=conv2d(weight_shape=(3,3,64,64),strides=(1,1),padding='SAME',activation='relu') + self.conv1=conv2d(32,[3,3],3,strides=(1,1),padding='SAME',activation='relu') + self.conv2=conv2d(64,[3,3],32,strides=(1,1),padding='SAME',activation='relu') + self.conv3=conv2d(64,[3,3],64,strides=(1,1),padding='SAME',activation='relu') # Create two dense layers with relu and linear activations - self.dense1=dense([64*4*4,64],activation='relu') - self.dense2=dense([64,10]) + self.dense1=dense(64,64*4*4,activation='relu') + self.dense2=dense(10,64) # Store the parameters of the layers in a list self.param = [self.conv1.param, self.conv2.param, From fa1c6da63f8fc246d8c7ea846f94fab0d63ed706 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Tue, 29 Aug 2023 16:34:09 +0800 Subject: [PATCH 191/337] Add files via upload --- 7.0/neuralnetwork/DL/tensorflow/parallel/cnn.py | 12 ++++++------ 7.0/neuralnetwork/DL/tensorflow/parallel/cnn_lmix.py | 12 ++++++------ 7.0/neuralnetwork/DL/tensorflow/parallel/nn.py | 4 ++-- 7.0/neuralnetwork/DL/tensorflow/parallel/nn_acc.py | 4 ++-- .../DL/tensorflow/parallel/nn_attenuate.py | 4 ++-- .../DL/tensorflow/parallel/nn_clipping.py | 4 ++-- .../DL/tensorflow/parallel/nn_device.py | 4 ++-- 7 files changed, 22 insertions(+), 22 deletions(-) diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/cnn.py b/7.0/neuralnetwork/DL/tensorflow/parallel/cnn.py index f99003e0..efe89c5c 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/cnn.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/cnn.py @@ -16,12 +16,12 @@ def __init__(self): def build(self): # Create three convolutional layers with relu activations - self.conv1=conv2d(weight_shape=(3,3,3,32),strides=(1,1),padding='SAME',activation='relu') - self.conv2=conv2d(weight_shape=(3,3,32,64),strides=(2,2),padding='VALID',activation='relu') - self.conv3=conv2d(weight_shape=(3,3,64,64),strides=(1,1),padding='SAME',activation='relu') + self.conv1=conv2d(32,[3,3],3,strides=(1,1),padding='SAME',activation='relu') + self.conv2=conv2d(64,[3,3],32,strides=(2,2),padding='SAME',activation='relu') + self.conv3=conv2d(64,[3,3],64,strides=(1,1),padding='SAME',activation='relu') # Create two dense layers with relu and linear activations - self.dense1=dense([64*4*4,64],activation='relu') - self.dense2=dense([64,10]) + self.dense1=dense(64,64*4*4,activation='relu') + self.dense2=dense(10,64) # Store the parameters of the layers in a list self.param = [self.conv1.param, self.conv2.param, @@ -55,4 +55,4 @@ def loss(self,output,labels): def opt(self,gradient): # Perform optimization on the parameters using the gradient and the batch count param=self.optimizer.opt(gradient,self.param,self.bc[0]) - return param + return param \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/cnn_lmix.py b/7.0/neuralnetwork/DL/tensorflow/parallel/cnn_lmix.py index 5776dde6..c4bfd3ed 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/cnn_lmix.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/cnn_lmix.py @@ -19,12 +19,12 @@ def __init__(self): def build(self): # Create three convolutional layers with relu activations - self.conv1=conv2d(weight_shape=(3,3,3,32),strides=(1,1),padding='SAME',activation='relu') - self.conv2=conv2d(weight_shape=(3,3,32,64),strides=(1,1),padding='SAME',activation='relu') - self.conv3=conv2d(weight_shape=(3,3,64,64),strides=(1,1),padding='SAME',activation='relu') + self.conv1=conv2d(32,[3,3],3,strides=(1,1),padding='SAME',activation='relu') + self.conv2=conv2d(64,[3,3],32,strides=(2,2),padding='SAME',activation='relu') + self.conv3=conv2d(64,[3,3],64,strides=(1,1),padding='SAME',activation='relu') # Create two dense layers with relu and linear activations - self.dense1=dense([64*4*4,64],activation='relu') - self.dense2=dense([64,10]) + self.dense1=dense(64,64*4*4,activation='relu') + self.dense2=dense(10,64) # Store the parameters of the layers in a list self.param = [self.conv1.param, self.conv2.param, @@ -63,4 +63,4 @@ def loss(self,output,labels): def opt(self,gradient): # Perform optimization on the parameters using the gradient and the batch count param=self.optimizer.opt(gradient,self.param,self.bc[0]) - return param + return param \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn.py index 1b16b5c0..d4aeec3a 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/nn.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/nn.py @@ -13,8 +13,8 @@ def __init__(self): def build(self): # Create two dense layers with relu and linear activations - self.layer1=d.dense([784,128],activation='relu') - self.layer2=d.dense([128,10]) + self.layer1=d.dense(128,784,activation='relu') + self.layer2=d.dense(10,128) # Store the parameters of the layers in a list self.param=[self.layer1.param,self.layer2.param] return diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_acc.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_acc.py index 19b055dc..a9ff8958 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_acc.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_acc.py @@ -14,8 +14,8 @@ def __init__(self): def build(self): # Create two dense layers with relu and linear activations - self.layer1=d.dense([784,128],activation='relu') - self.layer2=d.dense([128,10]) + self.layer1=d.dense(128,784,activation='relu') + self.layer2=d.dense(10,128) # Store the parameters of the layers in a list self.param=[self.layer1.param,self.layer2.param] return diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_attenuate.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_attenuate.py index 2f176151..43ac8902 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_attenuate.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_attenuate.py @@ -17,8 +17,8 @@ def __init__(self): def build(self): # Create two dense layers with relu and linear activations - self.layer1=d.dense([784,128],activation='relu') - self.layer2=d.dense([128,10]) + self.layer1=d.dense(128,784,activation='relu') + self.layer2=d.dense(10,128) # Store the parameters of the layers in a list self.param=[self.layer1.param,self.layer2.param] return diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_clipping.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_clipping.py index a0f757d3..9b5d973a 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_clipping.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_clipping.py @@ -14,8 +14,8 @@ def __init__(self): def build(self): # Create two dense layers with relu and linear activations - self.layer1=d.dense([784,128],activation='relu') - self.layer2=d.dense([128,10]) + self.layer1=d.dense(128,784,activation='relu') + self.layer2=d.dense(10,128) # Store the parameters of the layers in a list self.param=[self.layer1.param,self.layer2.param] return diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_device.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_device.py index e48acac3..a4fe0bfb 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_device.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_device.py @@ -13,8 +13,8 @@ def __init__(self): # initialize the network def build(self): # build function, kernel uses it to create the network layers # Create two dense layers with relu and linear activations - self.layer1=d.dense([784,128],activation='relu') # the first layer with 784 input units and 128 output units and ReLU activation - self.layer2=d.dense([128,10]) # the second layer with 128 input units and 10 output units and linear activation + self.layer1=d.dense(128,784,activation='relu') # the first layer with 784 input units and 128 output units and ReLU activation + self.layer2=d.dense(10,128) # the second layer with 128 input units and 10 output units and linear activation # Store the parameters of the layers in a list self.param=[self.layer1.param,self.layer2.param] # parameter list of both layers, kernel uses it list for backpropagation return From 0b201670116f35d0a767eded469a8dcd04502c45 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 31 Aug 2023 23:07:55 +0800 Subject: [PATCH 192/337] Delete 7.0/neuralnetwork/DL/tensorflow/layer/attn_LSTM.py --- .../DL/tensorflow/layer/attn_LSTM.py | 44 ------------------- 1 file changed, 44 deletions(-) delete mode 100644 7.0/neuralnetwork/DL/tensorflow/layer/attn_LSTM.py diff --git a/7.0/neuralnetwork/DL/tensorflow/layer/attn_LSTM.py b/7.0/neuralnetwork/DL/tensorflow/layer/attn_LSTM.py deleted file mode 100644 index 1d41befd..00000000 --- a/7.0/neuralnetwork/DL/tensorflow/layer/attn_LSTM.py +++ /dev/null @@ -1,44 +0,0 @@ -import tensorflow as tf -from Note.nn.layer.LSTM import LSTM -from Note.nn.layer.dense import dense -from Note.nn.layer.attention import attention - -# Define a recurrent neural network class with LSTM layers and attention -class attn_lstm: - def __init__(self): - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.opt=tf.keras.optimizers.Adam() - - - def build(self): - # Create two LSTM layers with 50 hidden units each - self.lstm1=LSTM(weight_shape=(3,50),return_sequence=True) # First LSTM layer that returns the full sequence - self.lstm2=LSTM(weight_shape=(50,50),) # Second LSTM layer that returns the last output only - # Create a dense layer with 10 output units and softmax activation - self.dense=dense([100, 10],activation='softmax') - # Create an attention object with weight shape (50, 50) - self.attn=attention((50,50)) - # Store the parameters of the layers and the attention in a list - self.param=[self.lstm1.param, - self.lstm2.param, - self.dense.param, - self.attn.param] # Add the attention parameters to the list - return - - - def fp(self,data): - # Perform forward propagation on the input data - x=self.lstm1.output(data) # First LSTM layer output, shape (B, N, 50) - y=self.lstm2.output(x) # Second LSTM layer output, shape (B, 50) - # Use the attention object to compute the context vector from x and y, shape (B, 50) - context_vector,_,_=self.attn.output(x,y) - # Concatenate the context vector and the second LSTM output, shape (B, 100) - z=tf.concat([context_vector,y],axis=-1) - output=self.dense.output(z) # Dense layer output with softmax activation, shape (B, 10) - return output - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) \ No newline at end of file From be3c4328494f9fbae22416b38b171b91b0d7ff72 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 31 Aug 2023 23:08:34 +0800 Subject: [PATCH 193/337] Update self_LSTM.py --- 7.0/neuralnetwork/DL/tensorflow/layer/self_LSTM.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/7.0/neuralnetwork/DL/tensorflow/layer/self_LSTM.py b/7.0/neuralnetwork/DL/tensorflow/layer/self_LSTM.py index de64b987..59d70143 100644 --- a/7.0/neuralnetwork/DL/tensorflow/layer/self_LSTM.py +++ b/7.0/neuralnetwork/DL/tensorflow/layer/self_LSTM.py @@ -17,13 +17,13 @@ def __init__(self,vocab_size,embed_size,num_heads,num_layers): def build(self): # Create an embedding layer to map input tokens to vectors - self.embedding_layer = embedding(self.vocab_size,self.embed_size) + self.embedding_layer = embedding(self.embed_size,self.vocab_size) # Create a LSTM layer to process the input sequence - self.lstm_layer = LSTM(weight_shape=[self.embed_size,self.embed_size],return_sequence=True) + self.lstm_layer = LSTM(self.embed_size,self.embed_size,return_sequence=True) # Create a self attention layer to attend to the LSTM output - self.self_attention_layer = self_attention(weight_shape=[self.embed_size,self.embed_size],num_heads=self.num_heads) + self.self_attention_layer = self_attention(self.embed_size,self.embed_size,num_heads=self.num_heads) # Create a linear layer to map the attention output to logits - self.logits_layer = dense([self.embed_size,self.vocab_size]) + self.logits_layer = dense(self.vocab_size，self.embed_size) # Store the parameters of the layers in a list self.param = [self.embedding_layer.param] self.param.extend(self.lstm_layer.param) From ebcd6bbdf239535c3138085b0a540c07ea9a25e1 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 31 Aug 2023 23:08:46 +0800 Subject: [PATCH 194/337] Update LSTM.py --- 7.0/neuralnetwork/DL/tensorflow/layer/LSTM.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/7.0/neuralnetwork/DL/tensorflow/layer/LSTM.py b/7.0/neuralnetwork/DL/tensorflow/layer/LSTM.py index 272855a3..4577ba88 100644 --- a/7.0/neuralnetwork/DL/tensorflow/layer/LSTM.py +++ b/7.0/neuralnetwork/DL/tensorflow/layer/LSTM.py @@ -12,10 +12,10 @@ def __init__(self): def build(self): # Create two LSTM layers with 50 hidden units each - self.lstm1=LSTM(weight_shape=(3,50),return_sequence=True) # First LSTM layer that returns the full sequence - self.lstm2=LSTM(weight_shape=(50,50),) # Second LSTM layer that returns the last output only + self.lstm1=LSTM(50,3,return_sequence=True) # First LSTM layer that returns the full sequence + self.lstm2=LSTM(50,50) # Second LSTM layer that returns the last output only # Create a dense layer with 10 output units - self.dense=dense([50, 10]) + self.dense=dense(10,50) # Store the parameters of the layers in a list self.param=[self.lstm1.param, self.lstm2.param, @@ -34,4 +34,4 @@ def fp(self,data): def loss(self, output, labels): # Compute the loss between the output and the labels - return self.loss_object(labels, output) \ No newline at end of file + return self.loss_object(labels, output) From 65c3b725620e57af55343887675db75f604f5b27 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 31 Aug 2023 23:09:00 +0800 Subject: [PATCH 195/337] Delete 7.0/neuralnetwork/DL/tensorflow/parallel/attn_LSTM.py --- .../DL/tensorflow/parallel/attn_LSTM.py | 53 ------------------- 1 file changed, 53 deletions(-) delete mode 100644 7.0/neuralnetwork/DL/tensorflow/parallel/attn_LSTM.py diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/attn_LSTM.py b/7.0/neuralnetwork/DL/tensorflow/parallel/attn_LSTM.py deleted file mode 100644 index 198e46b1..00000000 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/attn_LSTM.py +++ /dev/null @@ -1,53 +0,0 @@ -import tensorflow as tf -from Note.nn.layer.LSTM import LSTM -from Note.nn.layer.dense import dense -from Note.nn.layer.attention import attention -from Note.nn.parallel.optimizer import Adam - -# Define a recurrent neural network class with LSTM layers and attention -class attn_lstm: - def __init__(self): - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.optimizer=Adam() - # Initialize a variable to keep track of the batch count - self.bc=tf.Variable(0, dtype=tf.float32) - - - def build(self): - # Create two LSTM layers with 50 hidden units each - self.lstm1=LSTM(weight_shape=(3,50),return_sequence=True) # First LSTM layer that returns the full sequence - self.lstm2=LSTM(weight_shape=(50,50),) # Second LSTM layer that returns the last output only - # Create a dense layer with 10 output units and softmax activation - self.dense=dense([100, 10],activation='softmax') - # Create an attention object with weight shape (50, 50) - self.attn=attention((50,50)) - # Store the parameters of the layers and the attention in a list - self.param=[self.lstm1.param, - self.lstm2.param, - self.dense.param, - self.attn.param] # Add the attention parameters to the list - return - - - def fp(self,data): - # Perform forward propagation on the input data - x=self.lstm1.output(data) # First LSTM layer output, shape (B, N, 50) - y=self.lstm2.output(x) # Second LSTM layer output, shape (B, 50) - # Use the attention object to compute the context vector from x and y, shape (B, 50) - context_vector,_,_=self.attn.output(x,y) - # Concatenate the context vector and the second LSTM output, shape (B, 100) - z=tf.concat([context_vector,y],axis=-1) - output=self.dense.output(z) # Dense layer output with softmax activation, shape (B, 10) - return output - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) - - - def opt(self,gradient): - # Perform optimization on the parameters using the gradient and the batch count - param=self.optimizer.opt(gradient,self.param,self.bc[0]) - return param \ No newline at end of file From 5899d2d9c96d386d1b75d3e16077d06a3dabe36e Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 31 Aug 2023 23:10:03 +0800 Subject: [PATCH 196/337] Update self_LSTM.py --- 7.0/neuralnetwork/DL/tensorflow/parallel/self_LSTM.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/self_LSTM.py b/7.0/neuralnetwork/DL/tensorflow/parallel/self_LSTM.py index 72fd6687..784d4c00 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/self_LSTM.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/self_LSTM.py @@ -20,13 +20,13 @@ def __init__(self,vocab_size,embed_size,num_heads,num_layers): def build(self): # Create an embedding layer to map input tokens to vectors - self.embedding_layer = embedding(self.vocab_size,self.embed_size) + self.embedding_layer = embedding(self.embed_size,self.vocab_size) # Create a LSTM layer to process the input sequence - self.lstm_layer = LSTM(weight_shape=[self.embed_size,self.embed_size],return_sequence=True) + self.lstm_layer = LSTM(self.embed_size,self.embed_size,return_sequence=True) # Create a self attention layer to attend to the LSTM output - self.self_attention_layer = self_attention(weight_shape=[self.embed_size,self.embed_size],num_heads=self.num_heads) + self.self_attention_layer = self_attention(self.embed_size,self.embed_size,num_heads=self.num_heads) # Create a linear layer to map the attention output to logits - self.logits_layer = dense([self.embed_size,self.vocab_size]) + self.logits_layer = dense(self.vocab_size，self.embed_size) # Store the parameters of the layers in a list self.param = [self.embedding_layer.param] self.param.extend(self.lstm_layer.param) From d081636dfc503c0899bf48a7f5be51a07027e04d Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 31 Aug 2023 23:10:16 +0800 Subject: [PATCH 197/337] Update LSTM.py --- 7.0/neuralnetwork/DL/tensorflow/parallel/LSTM.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/LSTM.py b/7.0/neuralnetwork/DL/tensorflow/parallel/LSTM.py index e8686c10..c9d9bddd 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/LSTM.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/LSTM.py @@ -15,10 +15,10 @@ def __init__(self): def build(self): # Create two LSTM layers with 50 hidden units each - self.lstm1=LSTM(weight_shape=(3,50),return_sequence=True) # First LSTM layer that returns the full sequence - self.lstm2=LSTM(weight_shape=(50,50),) # Second LSTM layer that returns the last output only + self.lstm1=LSTM(50,3,return_sequence=True) # First LSTM layer that returns the full sequence + self.lstm2=LSTM(50,50) # Second LSTM layer that returns the last output only # Create a dense layer with 10 output units - self.dense=dense([50, 10]) + self.dense=dense(10,50) # Store the parameters of the layers in a list self.param=[self.lstm1.param, self.lstm2.param, @@ -43,4 +43,4 @@ def loss(self, output, labels): def opt(self,gradient): # Perform optimization on the parameters using the gradient and the batch count param=self.optimizer.opt(gradient,self.param,self.bc[0]) - return param \ No newline at end of file + return param From d2d1119d7cd5c4d8bd33832a6e05341b558697a1 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Wed, 6 Sep 2023 13:49:49 +0800 Subject: [PATCH 198/337] Update nn_device.py --- 7.0/neuralnetwork/DL/tensorflow/parallel/nn_device.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_device.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_device.py index a4fe0bfb..def5e50d 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_device.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_device.py @@ -33,8 +33,16 @@ def loss(self,output,labels,p): # loss function, kernel uses it to calculate los return self.loss_object(labels,output) # return the mean softmax cross entropy loss between labels and output + def GradientTape(self,data,labels,p): + with tf.device(assign_device(p,'GPU')): + with tf.GradientTape(persistent=True) as tape: + output=self.fp(data,p) + loss=self.loss(output,labels,p) + return tape,output,loss + + def opt(self,gradient,p): # optimization function, kernel uses it to optimize parameter # Perform optimization on the parameters using the gradient with tf.device(assign_device(p,'GPU')): # assign the device according to the process index p param=self.optimizer.opt(gradient,self.param) # apply the Note's momentum optimizer to update the parameters using the gradient - return param \ No newline at end of file + return param From 43a7434ea72954ad23f4a64f95c53a74f029aba6 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Sat, 9 Sep 2023 18:09:15 +0800 Subject: [PATCH 199/337] Delete 7.0/neuralnetwork/DL/tensorflow/parallel/self_LSTM.py --- .../DL/tensorflow/parallel/self_LSTM.py | 54 ------------------- 1 file changed, 54 deletions(-) delete mode 100644 7.0/neuralnetwork/DL/tensorflow/parallel/self_LSTM.py diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/self_LSTM.py b/7.0/neuralnetwork/DL/tensorflow/parallel/self_LSTM.py deleted file mode 100644 index 784d4c00..00000000 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/self_LSTM.py +++ /dev/null @@ -1,54 +0,0 @@ -import tensorflow as tf -from Note.nn.parallel.optimizer import Adam -from Note.nn.layer.LSTM import LSTM -from Note.nn.layer.self_attention import self_attention -from Note.nn.layer.dense import dense -from Note.nn.layer.emdedding import embedding - -# Define a LSTM self attention neural network class -class self_LSTM: - def __init__(self,vocab_size,embed_size,num_heads,num_layers): - self.vocab_size=vocab_size - self.embed_size=embed_size - self.num_heads=num_heads - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.optimizer=Adam() - # Initialize a variable to keep track of the batch count - self.bc=tf.Variable(0,dtype=tf.float32) - - - def build(self): - # Create an embedding layer to map input tokens to vectors - self.embedding_layer = embedding(self.embed_size,self.vocab_size) - # Create a LSTM layer to process the input sequence - self.lstm_layer = LSTM(self.embed_size,self.embed_size,return_sequence=True) - # Create a self attention layer to attend to the LSTM output - self.self_attention_layer = self_attention(self.embed_size,self.embed_size,num_heads=self.num_heads) - # Create a linear layer to map the attention output to logits - self.logits_layer = dense(self.vocab_size，self.embed_size) - # Store the parameters of the layers in a list - self.param = [self.embedding_layer.param] - self.param.extend(self.lstm_layer.param) - self.param.extend(self.self_attention_layer.param) - self.param.extend(self.logits_layer.param) - - - def fp(self,data): - # Perform forward propagation on the input data - x = self.embedding_layer.output(data) # shape: (batch_size, seq_len, embed_size) - x = self.lstm_layer.output(x) # shape: (batch_size, seq_len, embed_size) - x, attention_weights = self.self_attention_layer.output(x,self.num_heads) # shape: (batch_size, seq_len, embed_size) - output = self.logits_layer.output(x) # shape: (batch_size, seq_len, vocab_size) - return output - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) - - - def opt(self,gradient): - # Perform optimization on the parameters using the gradient and the batch count - param=self.optimizer.opt(gradient,self.param,self.bc[0]) - return param From 3a1aa01a924e44c9ff960bf38b7c37dfe08feed4 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Sat, 9 Sep 2023 18:09:30 +0800 Subject: [PATCH 200/337] Delete 7.0/neuralnetwork/DL/tensorflow/layer/self_LSTM.py --- .../DL/tensorflow/layer/self_LSTM.py | 45 ------------------- 1 file changed, 45 deletions(-) delete mode 100644 7.0/neuralnetwork/DL/tensorflow/layer/self_LSTM.py diff --git a/7.0/neuralnetwork/DL/tensorflow/layer/self_LSTM.py b/7.0/neuralnetwork/DL/tensorflow/layer/self_LSTM.py deleted file mode 100644 index 59d70143..00000000 --- a/7.0/neuralnetwork/DL/tensorflow/layer/self_LSTM.py +++ /dev/null @@ -1,45 +0,0 @@ -import tensorflow as tf -from Note.nn.layer.LSTM import LSTM -from Note.nn.layer.self_attention import self_attention -from Note.nn.layer.dense import dense -from Note.nn.layer.emdedding import embedding - -# Define a LSTM self attention neural network class -class self_LSTM: - def __init__(self,vocab_size,embed_size,num_heads,num_layers): - self.vocab_size=vocab_size - self.embed_size=embed_size - self.num_heads=num_heads - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.opt=tf.keras.optimizers.Adam() - - - def build(self): - # Create an embedding layer to map input tokens to vectors - self.embedding_layer = embedding(self.embed_size,self.vocab_size) - # Create a LSTM layer to process the input sequence - self.lstm_layer = LSTM(self.embed_size,self.embed_size,return_sequence=True) - # Create a self attention layer to attend to the LSTM output - self.self_attention_layer = self_attention(self.embed_size,self.embed_size,num_heads=self.num_heads) - # Create a linear layer to map the attention output to logits - self.logits_layer = dense(self.vocab_size，self.embed_size) - # Store the parameters of the layers in a list - self.param = [self.embedding_layer.param] - self.param.extend(self.lstm_layer.param) - self.param.extend(self.self_attention_layer.param) - self.param.extend(self.logits_layer.param) - - - def fp(self,data): - # Perform forward propagation on the input data - x = self.embedding_layer.output(data) # shape: (batch_size, seq_len, embed_size) - x = self.lstm_layer.output(x) # shape: (batch_size, seq_len, embed_size) - x, attention_weights = self.self_attention_layer.output(x,self.num_heads) # shape: (batch_size, seq_len, embed_size) - output = self.logits_layer.output(x) # shape: (batch_size, seq_len, vocab_size) - return output - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) From beec5967e22b24f1cbd18882ac7f953c7d6ad061 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Sun, 10 Sep 2023 21:30:16 +0800 Subject: [PATCH 201/337] Add files via upload --- .../RL/tensorflow/parallel/DDPG.py | 84 +++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 7.0/neuralnetwork/RL/tensorflow/parallel/DDPG.py diff --git a/7.0/neuralnetwork/RL/tensorflow/parallel/DDPG.py b/7.0/neuralnetwork/RL/tensorflow/parallel/DDPG.py new file mode 100644 index 00000000..2705adbb --- /dev/null +++ b/7.0/neuralnetwork/RL/tensorflow/parallel/DDPG.py @@ -0,0 +1,84 @@ +import tensorflow as tf # import TensorFlow library +import numpy as np # import NumPy library +import gym # import OpenAI Gym library + + +class actor: # define a class for the actor network + def __init__(self,state_dim,hidden_dim,action_dim,action_bound): # initialize the network with state dimension, hidden dimension, action dimension and action bound + self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) # create a weight matrix for the first layer + self.bias1=tf.Variable(tf.random.normal([hidden_dim])) # create a bias vector for the first layer + self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) # create a weight matrix for the second layer + self.bias2=tf.Variable(tf.random.normal([action_dim])) # create a bias vector for the second layer + self.action_bound=action_bound # store the action bound + self.param=[self.weight1,self.bias1,self.weight2,self.bias2] # store the network parameters in a list + + + def fp(self,x): # forward propagation function, kernel uses it for forward propagation + x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) # apply the first layer with ReLU activation + return tf.nn.tanh(tf.matmul(x,self.weight2)+self.bias2)*self.action_bound # apply the second layer with tanh activation and scale it by the action bound + + +class critic: # define a class for the critic network + def __init__(self,state_dim,hidden_dim,action_dim): # initialize the network with state dimension, hidden dimension and action dimension + self.weight1=tf.Variable(tf.random.normal([state_dim+action_dim,hidden_dim])) # create a weight matrix for the first layer + self.bias1=tf.Variable(tf.random.normal([hidden_dim])) # create a bias vector for the first layer + self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) # create a weight matrix for the second layer + self.bias2=tf.Variable(tf.random.normal([action_dim])) # create a bias vector for the second layer + self.param=[self.weight1,self.bias1,self.weight2,self.bias2] # store the network parameters in a list + + + def fp(self,x,a): # forward propagation function, kernel uses it for forward propagation + cat=tf.concat([x,a],axis=1) # concatenate the state and action vectors along the first axis + x=tf.nn.relu(tf.matmul(cat,self.weight1)+self.bias1) # apply the first layer with ReLU activation + return tf.matmul(x,self.weight2)+self.bias2 # apply the second layer and return the output + + +class DDPG: # define a class for the DDPG agent + def __init__(self,hidden_dim,sigma,gamma,tau,actor_lr,critic_lr): # initialize the agent with hidden dimension, noise scale, discount factor, soft update factor, actor learning rate and critic learning rate + self.genv=[gym.make('Pendulum-v0') for _ in range(5)] # create a list of 5 environments + state_dim=self.genv.observation_space.shape[0] # get the state dimension from the environment observation space + action_dim=self.genv.action_space.shape[0] # get the action dimension from the environment action space + action_bound=self.genv.action_space.high[0] # get the action bound from the environment action space high value (assume symmetric) + self.actor=actor(state_dim,hidden_dim,action_dim,action_bound) # create an actor network for the agent + self.critic=critic(state_dim,hidden_dim,action_dim) # create a critic network for the agent + self.target_actor=actor(state_dim,hidden_dim,action_dim,action_bound) # create a target actor network for the agent + self.target_critic=critic(state_dim,hidden_dim,action_dim) # create a target critic network for the agent + self.actor_param=self.actor.param # parameter list of actor network, kernel uses it list for backpropagation + self.critic_param=self.critic.param # parameter list of critic network, kernel uses it list for backpropagation + self.target_actor.param=self.actor_param.copy() # copy the parameters from actor network to target actor network + self.target_critic.param=self.critic_param.copy() # copy the parameters from critic network to target critic network + self.param=[self.actor_param,self.critic_param] # parameter list of both networks, kernel uses it list for backpropagation + self.sigma=sigma # noise scale + self.gamma=gamma # discount factor + self.tau=tau # soft update factor + self.opt=tf.keras.optimizers.Adam() # optimizer, kernel uses it to optimize. Here we use Adam optimizer + + + def noise(self): # noise function, kernel uses it to generate exploration noise + return np.random.normal(scale=self.sigma) # return a random sample from a normal distribution with zero mean and sigma scale + + + def env(self,a=None,p=None,initial=None): # environment function, kernel uses it to interact with the environment + if initial==True: # if initial is True, reset the environment with index p and return the initial state + state=self.genv[p].reset(seed=0) + return state + else: # otherwise, take an action and return the next state, reward and done flag from the environment with index p + next_state,reward,done,_=self.genv[p].step(a) + return next_state,reward,done + + + def loss(self,s,a,next_s,r,d): # loss function, kernel uses it to calculate loss + a=tf.expand_dims(a,axis=1) # expand the action vector to match the Q-value matrix shape + next_q_value=self.target_critic.fp(next_s,self.target_actor.fp(next_s)) # get the Q-value for the next state and action from the target critic network + q_target=r+self.gamma*next_q_value*(1-d) # calculate the target value using Bellman equation with discount factor gamma + actor_loss=-tf.reduce_mean(self.critic.fp(s,self.actor.fp(s))) # calculate the actor loss as the negative mean of the Q-value for the current state and action from the critic network + critic_loss=tf.reduce_mean((self.critic.fp(s,a)-q_target)**2) # calculate the critic loss as the mean squared error between Q-value and target value + return [actor_loss,critic_loss] # return a list of actor loss and critic loss + + + def update_param(self): # update function, kernel uses it to update parameter + for target_param,param in zip(self.target_actor.param,self.actor.param): # for each pair of parameters in target actor network and actor network + target_param=target_param*(1.0-self.tau)+param*self.tau # update the target parameter using soft update with factor tau + for target_param,param in zip(self.target_critic.param,self.critic.param): # for each pair of parameters in target critic network and critic network + target_param=target_param*(1.0-self.tau)+param*self.tau # update the target parameter using soft update with factor tau + return \ No newline at end of file From dd59cfcea79dd4cf1634fcd81344404d06cf6e24 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Tue, 12 Sep 2023 20:35:05 +0800 Subject: [PATCH 202/337] Delete 7.0/neuralnetwork/DL/tensorflow/layer directory --- 7.0/neuralnetwork/DL/tensorflow/layer/LSTM.py | 37 ------------ 7.0/neuralnetwork/DL/tensorflow/layer/cnn.py | 49 ---------------- .../DL/tensorflow/layer/cnn_lmix.py | 57 ------------------- 7.0/neuralnetwork/DL/tensorflow/layer/nn.py | 27 --------- .../DL/tensorflow/layer/nn_acc.py | 32 ----------- .../DL/tensorflow/layer/nn_clipping.py | 40 ------------- .../DL/tensorflow/layer/transformer.py | 46 --------------- 7 files changed, 288 deletions(-) delete mode 100644 7.0/neuralnetwork/DL/tensorflow/layer/LSTM.py delete mode 100644 7.0/neuralnetwork/DL/tensorflow/layer/cnn.py delete mode 100644 7.0/neuralnetwork/DL/tensorflow/layer/cnn_lmix.py delete mode 100644 7.0/neuralnetwork/DL/tensorflow/layer/nn.py delete mode 100644 7.0/neuralnetwork/DL/tensorflow/layer/nn_acc.py delete mode 100644 7.0/neuralnetwork/DL/tensorflow/layer/nn_clipping.py delete mode 100644 7.0/neuralnetwork/DL/tensorflow/layer/transformer.py diff --git a/7.0/neuralnetwork/DL/tensorflow/layer/LSTM.py b/7.0/neuralnetwork/DL/tensorflow/layer/LSTM.py deleted file mode 100644 index 4577ba88..00000000 --- a/7.0/neuralnetwork/DL/tensorflow/layer/LSTM.py +++ /dev/null @@ -1,37 +0,0 @@ -import tensorflow as tf -from Note.nn.layer.LSTM import LSTM -from Note.nn.layer.dense import dense - -# Define a recurrent neural network class with LSTM layers -class lstm: - def __init__(self): - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.opt=tf.keras.optimizers.Adam() - - - def build(self): - # Create two LSTM layers with 50 hidden units each - self.lstm1=LSTM(50,3,return_sequence=True) # First LSTM layer that returns the full sequence - self.lstm2=LSTM(50,50) # Second LSTM layer that returns the last output only - # Create a dense layer with 10 output units - self.dense=dense(10,50) - # Store the parameters of the layers in a list - self.param=[self.lstm1.param, - self.lstm2.param, - self.dense.param, - ] - return - - - def fp(self,data): - # Perform forward propagation on the input data - x=self.lstm1.output(data) # First LSTM layer output - x=self.lstm2.output(x) # Second LSTM layer output - output=self.dense.output(x) # Dense layer output - return output - - - def loss(self, output, labels): - # Compute the loss between the output and the labels - return self.loss_object(labels, output) diff --git a/7.0/neuralnetwork/DL/tensorflow/layer/cnn.py b/7.0/neuralnetwork/DL/tensorflow/layer/cnn.py deleted file mode 100644 index 44ed6393..00000000 --- a/7.0/neuralnetwork/DL/tensorflow/layer/cnn.py +++ /dev/null @@ -1,49 +0,0 @@ -import tensorflow as tf -from Note.nn.layer.conv2d import conv2d -from Note.nn.layer.dense import dense -from Note.nn.layer.flatten import flatten - -# Define a convolutional neural network class -class cnn: - def __init__(self): - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.opt=tf.keras.optimizers.Adam() - - - def build(self): - # Create three convolutional layers with relu activations - self.conv1=conv2d(32,[3,3],3,strides=(1,1),padding='SAME',activation='relu') - self.conv2=conv2d(64,[3,3],32,strides=(1,1),padding='SAME',activation='relu') - self.conv3=conv2d(64,[3,3],64,strides=(1,1),padding='SAME',activation='relu') - # Create two dense layers with relu and linear activations - self.dense1=dense(64,64*4*4,activation='relu') - self.dense2=dense(10,64) - # Store the parameters of the layers in a list - self.param = [self.conv1.param, - self.conv2.param, - self.conv3.param, - self.dense1.param, - self.dense2.param, - ] - return - - - def fp(self,data): - # Perform forward propagation on the input data - x=self.conv1.output(data) # First convolutional layer - x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # First max pooling layer - x=self.conv2.output(x) # Second convolutional layer - x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # Second max pooling layer - x=self.conv3.output(x) # Third convolutional layer - x=flatten(x) # Flatten the output to a vector - x=tf.nn.relu(tf.matmul(x,self.dense1.weight)+self.dense1.bias) # First dense layer with relu activation - x=tf.nn.dropout(x,rate=0.5) # Apply dropout to prevent overfitting - output=tf.matmul(x,self.dense2.weight)+self.dense2.bias # Output layer with linear activation - output=tf.nn.dropout(output,rate=0.5) # Apply dropout to prevent overfitting - return output - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) diff --git a/7.0/neuralnetwork/DL/tensorflow/layer/cnn_lmix.py b/7.0/neuralnetwork/DL/tensorflow/layer/cnn_lmix.py deleted file mode 100644 index 76c97263..00000000 --- a/7.0/neuralnetwork/DL/tensorflow/layer/cnn_lmix.py +++ /dev/null @@ -1,57 +0,0 @@ -import tensorflow as tf -from Note.nn.layer.conv2d import conv2d -from Note.nn.layer.dense import dense -from Note.nn.layer.flatten import flatten -from Note.nn.layer.LMix import lmix - -# Define a convolutional neural network class with mixup augmentation -class cnn: - def __init__(self): - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.opt=tf.keras.optimizers.Adam() - # Initialize the alpha parameter for mixup - self.alpha=1.0 - - - def build(self): - # Create three convolutional layers with relu activations - self.conv1=conv2d(32,[3,3],3,strides=(1,1),padding='SAME',activation='relu') - self.conv2=conv2d(64,[3,3],32,strides=(1,1),padding='SAME',activation='relu') - self.conv3=conv2d(64,[3,3],64,strides=(1,1),padding='SAME',activation='relu') - # Create two dense layers with relu and linear activations - self.dense1=dense(64,64*4*4,activation='relu') - self.dense2=dense(10,64) - # Store the parameters of the layers in a list - self.param = [self.conv1.param, - self.conv2.param, - self.conv3.param, - self.dense1.param, - self.dense2.param, - ] - return - - - def data_func(self,data_batch,labels_batch): - # Apply mixup augmentation on the input data and labels - return lmix(data_batch,labels_batch,self.alpha) - - - def fp(self,data): - # Perform forward propagation on the input data - x=self.conv1.output(data) # First convolutional layer - x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # First max pooling layer - x=self.conv2.output(x) # Second convolutional layer - x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # Second max pooling layer - x=self.conv3.output(x) # Third convolutional layer - x=flatten(x) # Flatten the output to a vector - x=tf.nn.relu(tf.matmul(x,self.dense1.weight)+self.dense1.bias) # First dense layer with relu activation - x=tf.nn.dropout(x,rate=0.5) # Apply dropout to prevent overfitting - output=tf.matmul(x,self.dense2.weight)+self.dense2.bias # Output layer with linear activation - output=tf.nn.dropout(output,rate=0.5) # Apply dropout to prevent overfitting - return output - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) diff --git a/7.0/neuralnetwork/DL/tensorflow/layer/nn.py b/7.0/neuralnetwork/DL/tensorflow/layer/nn.py deleted file mode 100644 index 978e83c1..00000000 --- a/7.0/neuralnetwork/DL/tensorflow/layer/nn.py +++ /dev/null @@ -1,27 +0,0 @@ -import tensorflow as tf -import Note.nn.layer.dense as d -from Note.nn.layer.flatten import flatten - - -class nn: - def __init__(self): - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.opt=tf.keras.optimizers.Adam() - - - def build(self): - self.layer1=d.dense(128,784,activation='relu') - self.layer2=d.dense(10,128) - self.param=[self.layer1.param,self.layer2.param] - return - - - def fp(self,data): - data=flatten(data) - output1=self.layer1.output(data) - output2=self.layer2.output(output1) - return output2 - - - def loss(self,output,labels): - return self.loss_object(labels,output) diff --git a/7.0/neuralnetwork/DL/tensorflow/layer/nn_acc.py b/7.0/neuralnetwork/DL/tensorflow/layer/nn_acc.py deleted file mode 100644 index 771ad6ee..00000000 --- a/7.0/neuralnetwork/DL/tensorflow/layer/nn_acc.py +++ /dev/null @@ -1,32 +0,0 @@ -import tensorflow as tf -import Note.nn.layer.dense as d -from Note.nn.layer.flatten import flatten - - -class nn: - def __init__(self): - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.train_accuracy=tf.keras.metrics.SparseCategoricalAccuracy(name='train_accuracy') - self.opt=tf.keras.optimizers.Adam() - - - def build(self): - self.layer1=d.dense(128,784,activation='relu') - self.layer2=d.dense(10,128) - self.param=[self.layer1.param,self.layer2.param] - return - - - def fp(self,data): - data=flatten(data) - output1=self.layer1.output(data) - output2=self.layer2.output(output1) - return output2 - - - def loss(self,output,labels): - return self.loss_object(labels,output) - - - def accuracy(self,output,labels): - return self.train_accuracy(labels,output) diff --git a/7.0/neuralnetwork/DL/tensorflow/layer/nn_clipping.py b/7.0/neuralnetwork/DL/tensorflow/layer/nn_clipping.py deleted file mode 100644 index 02944c12..00000000 --- a/7.0/neuralnetwork/DL/tensorflow/layer/nn_clipping.py +++ /dev/null @@ -1,40 +0,0 @@ -import tensorflow as tf -import Note.nn.layer.dense as d -from Note.nn.layer.flatten import flatten -from Note.nn.gradient_clipping import gradient_clipping - -# Define a neural network class -class nn: - def __init__(self): - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.opt=tf.keras.optimizers.Adam() - - - def build(self): - # Create two dense layers with relu and linear activations - self.layer1=d.dense(128,784,activation='relu') - self.layer2=d.dense(10,128) - # Store the parameters of the layers in a list - self.param=[self.layer1.param,self.layer2.param] - return - - - def fp(self,data): - # Perform forward propagation on the input data - data=flatten(data) - output1=self.layer1.output(data) - output2=self.layer2.output(output1) - return output2 - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) - - - def gradient(self,tape,loss): - # Compute the gradients of the loss with respect to the parameters - grads=tape.gradient(loss,self.param) - # Clip the gradients by value to avoid exploding gradients - return gradient_clipping(grads,'value',1.0) diff --git a/7.0/neuralnetwork/DL/tensorflow/layer/transformer.py b/7.0/neuralnetwork/DL/tensorflow/layer/transformer.py deleted file mode 100644 index e431fc95..00000000 --- a/7.0/neuralnetwork/DL/tensorflow/layer/transformer.py +++ /dev/null @@ -1,46 +0,0 @@ -import tensorflow as tf -from Note.nn.layer.transformer import transformer -from Note.nn.layer.dense import dense -from Note.nn.layer.emdedding import embedding -from Note.nn.positional_encoding import positional_encoding - -# Define a positional encoding transformer neural network class -class Transformer: - def __init__(self,vocab_size,embed_size,num_heads,num_layers,max_len): - self.vocab_size=vocab_size - self.embed_size=embed_size - self.num_heads=num_heads - self.num_layers=num_layers - self.max_len=max_len - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.opt=tf.keras.optimizers.Adam() - - - def build(self): - # Create an embedding layer to map input tokens to vectors - self.embedding_layer = embedding(self.vocab_size,self.embed_size) - # Create a list of transformer layers - self.transformer_layers = [transformer(weight_shape=[self.embed_size, self.embed_size], num_heads=self.num_heads) for _ in range(self.num_layers)] - # Create a linear layer to map the transformer output to logits - self.logits_layer = dense([self.embed_size,self.vocab_size]) - # Store the parameters of the layers in a list - self.param = [self.embedding_layer.param] - for transformer_layer in self.transformer_layers: - self.param.extend(transformer_layer.param) - self.param.extend(self.logits_layer.param) - - - def fp(self,data): - # Perform forward propagation on the input data - x = self.embedding_layer.output(data) # shape: (batch_size, seq_len, embed_size) - x += positional_encoding(self.max_len,self.embed_size) # shape: (batch_size, seq_len, embed_size) - for transformer_layer in self.transformer_layers: - x = transformer_layer.output(x) # shape: (batch_size, seq_len, embed_size) - output = self.logits_layer.output(x) # shape: (batch_size, seq_len, vocab_size) - return output - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) From a80116f53737fecbeca5f586baecdbc8d37408c5 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 28 Sep 2023 14:38:35 +0800 Subject: [PATCH 203/337] Update nn_acc.py --- 7.0/neuralnetwork/DL/tensorflow/parallel/nn_acc.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_acc.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_acc.py index a9ff8958..4ffcf38e 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_acc.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_acc.py @@ -3,6 +3,7 @@ from Note.nn.parallel.optimizer import Momentum from Note.nn.layer.flatten import flatten from Note.nn.accuracy import sparse_categorical_accuracy +from Note.nn.Module import Module # Define a neural network class class nn: @@ -17,7 +18,7 @@ def build(self): self.layer1=d.dense(128,784,activation='relu') self.layer2=d.dense(10,128) # Store the parameters of the layers in a list - self.param=[self.layer1.param,self.layer2.param] + self.param=Module.param return @@ -41,4 +42,4 @@ def accuracy(self,output,labels): #accuracy function,kernel uses it to calculate def opt(self,gradient): # Perform optimization on the parameters using the gradient param=self.optimizer.opt(gradient,self.param) - return param \ No newline at end of file + return param From f1137fb84c1dc0c6aecf01a7fc23cb9d2aad0420 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 28 Sep 2023 14:38:49 +0800 Subject: [PATCH 204/337] Update nn_device.py --- 7.0/neuralnetwork/DL/tensorflow/parallel/nn_device.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_device.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_device.py index def5e50d..37a3cb3b 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_device.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_device.py @@ -3,6 +3,7 @@ from Note.nn.layer.flatten import flatten # import Note's flatten layer function from Note.nn.parallel.optimizer import Momentum # import Note's momentum optimizer module from Note.nn.parallel.assign_device import assign_device # import the function to assign device according to the process index and the device type +from Note.nn.Module import Module class nn: # A neural network class example, allocate device for multiple threads @@ -16,7 +17,7 @@ def build(self): # build function, kernel uses it to create the network layers self.layer1=d.dense(128,784,activation='relu') # the first layer with 784 input units and 128 output units and ReLU activation self.layer2=d.dense(10,128) # the second layer with 128 input units and 10 output units and linear activation # Store the parameters of the layers in a list - self.param=[self.layer1.param,self.layer2.param] # parameter list of both layers, kernel uses it list for backpropagation + self.param=Module.param # parameter list of both layers, kernel uses it list for backpropagation return From 90274324f153117cd57045eceb6d814cbbd371c4 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 28 Sep 2023 14:39:04 +0800 Subject: [PATCH 205/337] Update nn.py --- 7.0/neuralnetwork/DL/tensorflow/parallel/nn.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn.py index d4aeec3a..75f4df3c 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/nn.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/nn.py @@ -2,6 +2,7 @@ import Note.nn.layer.dense as d from Note.nn.parallel.optimizer import Momentum from Note.nn.layer.flatten import flatten +from Note.nn.Module import Module # Define a neural network class class nn: @@ -16,7 +17,7 @@ def build(self): self.layer1=d.dense(128,784,activation='relu') self.layer2=d.dense(10,128) # Store the parameters of the layers in a list - self.param=[self.layer1.param,self.layer2.param] + self.param=Module.param return @@ -36,4 +37,4 @@ def loss(self,output,labels): def opt(self,gradient): # Perform optimization on the parameters using the gradient param=self.optimizer.opt(gradient,self.param) - return param \ No newline at end of file + return param From 627192dd4172072f4c1f5f903aa7e8a301413080 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 28 Sep 2023 18:57:43 +0800 Subject: [PATCH 206/337] Update cnn.py --- 7.0/neuralnetwork/DL/tensorflow/parallel/cnn.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/cnn.py b/7.0/neuralnetwork/DL/tensorflow/parallel/cnn.py index efe89c5c..fc3182d1 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/cnn.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/cnn.py @@ -3,6 +3,7 @@ from Note.nn.layer.dense import dense from Note.nn.layer.flatten import flatten from Note.nn.parallel.optimizer import Adam +from Note.nn.Module import Module # Define a convolutional neural network class class cnn: @@ -23,12 +24,7 @@ def build(self): self.dense1=dense(64,64*4*4,activation='relu') self.dense2=dense(10,64) # Store the parameters of the layers in a list - self.param = [self.conv1.param, - self.conv2.param, - self.conv3.param, - self.dense1.param, - self.dense2.param, - ] + self.param=Module.param return @@ -55,4 +51,4 @@ def loss(self,output,labels): def opt(self,gradient): # Perform optimization on the parameters using the gradient and the batch count param=self.optimizer.opt(gradient,self.param,self.bc[0]) - return param \ No newline at end of file + return param From 0b892b5fef1b9f22132747eb2aaded4b225747e7 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 28 Sep 2023 18:57:56 +0800 Subject: [PATCH 207/337] Update cnn_lmix.py --- 7.0/neuralnetwork/DL/tensorflow/parallel/cnn_lmix.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/cnn_lmix.py b/7.0/neuralnetwork/DL/tensorflow/parallel/cnn_lmix.py index c4bfd3ed..822b03ed 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/cnn_lmix.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/cnn_lmix.py @@ -4,6 +4,7 @@ from Note.nn.layer.flatten import flatten from Note.nn.parallel.optimizer import Adam from Note.nn.layer.LMix import lmix +from Note.nn.Module import Module # Define a convolutional neural network class with mixup augmentation class cnn: @@ -26,12 +27,7 @@ def build(self): self.dense1=dense(64,64*4*4,activation='relu') self.dense2=dense(10,64) # Store the parameters of the layers in a list - self.param = [self.conv1.param, - self.conv2.param, - self.conv3.param, - self.dense1.param, - self.dense2.param, - ] + self.param=Module.param return @@ -63,4 +59,4 @@ def loss(self,output,labels): def opt(self,gradient): # Perform optimization on the parameters using the gradient and the batch count param=self.optimizer.opt(gradient,self.param,self.bc[0]) - return param \ No newline at end of file + return param From 60584ca62a817037a8db5854a560b56b0c04295d Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 28 Sep 2023 18:58:08 +0800 Subject: [PATCH 208/337] Update LSTM.py --- 7.0/neuralnetwork/DL/tensorflow/parallel/LSTM.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/LSTM.py b/7.0/neuralnetwork/DL/tensorflow/parallel/LSTM.py index c9d9bddd..7319a0e3 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/LSTM.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/LSTM.py @@ -2,6 +2,7 @@ from Note.nn.layer.LSTM import LSTM from Note.nn.layer.dense import dense from Note.nn.parallel.optimizer import Adam +from Note.nn.Module import Module # Define a recurrent neural network class with LSTM layers class lstm: @@ -20,10 +21,7 @@ def build(self): # Create a dense layer with 10 output units self.dense=dense(10,50) # Store the parameters of the layers in a list - self.param=[self.lstm1.param, - self.lstm2.param, - self.dense.param, - ] + self.param=Module.param return From a428eb59b5532f5d0649a56e9941e7b84576084c Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 28 Sep 2023 18:58:19 +0800 Subject: [PATCH 209/337] Update nn_attenuate.py --- 7.0/neuralnetwork/DL/tensorflow/parallel/nn_attenuate.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_attenuate.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_attenuate.py index 43ac8902..d81a059d 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_attenuate.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_attenuate.py @@ -4,6 +4,7 @@ from Note.nn.parallel.optimizer import Momentum from Note.nn.layer.flatten import flatten from Note.nn.accuracy import sparse_categorical_accuracy +from Note.nn.Module import Module # Define a neural network class with gradient attenuation class nn: @@ -20,7 +21,7 @@ def build(self): self.layer1=d.dense(128,784,activation='relu') self.layer2=d.dense(10,128) # Store the parameters of the layers in a list - self.param=[self.layer1.param,self.layer2.param] + self.param=Module.param return @@ -54,4 +55,4 @@ def attenuate(self,gradient,oc,p): #gradient attenuation function,kernel uses i def opt(self,gradient): # Perform optimization on the parameters using the gradient param=self.optimizer.opt(gradient,self.param) # Use the optimizer to update the parameters - return param \ No newline at end of file + return param From cbd57cc9be1158bca0e9d4d7077873ce8330e86f Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 28 Sep 2023 18:58:31 +0800 Subject: [PATCH 210/337] Update transformer.py --- 7.0/neuralnetwork/DL/tensorflow/parallel/transformer.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/transformer.py b/7.0/neuralnetwork/DL/tensorflow/parallel/transformer.py index d367c6ad..888e5369 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/transformer.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/transformer.py @@ -4,6 +4,7 @@ from Note.nn.layer.dense import dense from Note.nn.layer.emdedding import embedding from Note.nn.positional_encoding import positional_encoding +from Note.nn.Module import Module # Define a positional encoding transformer neural network class class Transformer: @@ -28,10 +29,7 @@ def build(self): # Create a linear layer to map the transformer output to logits self.logits_layer = dense([self.embed_size,self.vocab_size]) # Store the parameters of the layers in a list - self.param = [self.embedding_layer.param] - for transformer_layer in self.transformer_layers: - self.param.extend(transformer_layer.param) - self.param.extend(self.logits_layer.param) + self.param=Module.param def fp(self,data): From 587d0ee4a4c6b519cc350018e1d76502134cbd03 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 28 Sep 2023 18:58:42 +0800 Subject: [PATCH 211/337] Update nn_clipping.py --- 7.0/neuralnetwork/DL/tensorflow/parallel/nn_clipping.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_clipping.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_clipping.py index 9b5d973a..6a3c8e71 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_clipping.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_clipping.py @@ -3,6 +3,7 @@ from Note.nn.parallel.optimizer import Momentum from Note.nn.layer.flatten import flatten from Note.nn.gradient_clipping import gradient_clipping +from Note.nn.Module import Module # Define a neural network class class nn: @@ -17,7 +18,7 @@ def build(self): self.layer1=d.dense(128,784,activation='relu') self.layer2=d.dense(10,128) # Store the parameters of the layers in a list - self.param=[self.layer1.param,self.layer2.param] + self.param=Module.param return @@ -44,4 +45,4 @@ def gradient(self,tape,loss): def opt(self,gradient): # Perform optimization on the parameters using the gradient param=self.optimizer.opt(gradient,self.param) - return param \ No newline at end of file + return param From 73f46ce45388db57ee0330d98c10fbc50b1c87d2 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Fri, 29 Sep 2023 21:31:33 +0800 Subject: [PATCH 212/337] Delete 7.0/neuralnetwork/DL/tensorflow/parallel/transformer.py --- .../DL/tensorflow/parallel/transformer.py | 53 ------------------- 1 file changed, 53 deletions(-) delete mode 100644 7.0/neuralnetwork/DL/tensorflow/parallel/transformer.py diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/transformer.py b/7.0/neuralnetwork/DL/tensorflow/parallel/transformer.py deleted file mode 100644 index 888e5369..00000000 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/transformer.py +++ /dev/null @@ -1,53 +0,0 @@ -import tensorflow as tf -from Note.nn.parallel.optimizer import Adam -from Note.nn.layer.transformer import transformer -from Note.nn.layer.dense import dense -from Note.nn.layer.emdedding import embedding -from Note.nn.positional_encoding import positional_encoding -from Note.nn.Module import Module - -# Define a positional encoding transformer neural network class -class Transformer: - def __init__(self,vocab_size,embed_size,num_heads,num_layers,max_len): - self.vocab_size=vocab_size - self.embed_size=embed_size - self.num_heads=num_heads - self.num_layers=num_layers - self.max_len=max_len - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.optimizer=Adam() - # Initialize a variable to keep track of the batch count - self.bc=tf.Variable(0,dtype=tf.float32) - - - def build(self): - # Create an embedding layer to map input tokens to vectors - self.embedding_layer = embedding(self.vocab_size,self.embed_size) - # Create a list of transformer layers - self.transformer_layers = [transformer(weight_shape=[self.embed_size, self.embed_size], num_heads=self.num_heads) for _ in range(self.num_layers)] - # Create a linear layer to map the transformer output to logits - self.logits_layer = dense([self.embed_size,self.vocab_size]) - # Store the parameters of the layers in a list - self.param=Module.param - - - def fp(self,data): - # Perform forward propagation on the input data - x = self.embedding_layer.output(data) # shape: (batch_size, seq_len, embed_size) - x += positional_encoding(self.max_len,self.embed_size) # shape: (batch_size, seq_len, embed_size) - for transformer_layer in self.transformer_layers: - x = transformer_layer.output(x) # shape: (batch_size, seq_len, embed_size) - output = self.logits_layer.output(x) # shape: (batch_size, seq_len, vocab_size) - return output - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) - - - def opt(self,gradient): - # Perform optimization on the parameters using the gradient and the batch count - param=self.optimizer.opt(gradient,self.param,self.bc[0]) - return param From 8dff187d75924f24bb8bf7530e7fb25fc2791f27 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Fri, 29 Sep 2023 21:31:42 +0800 Subject: [PATCH 213/337] Delete 7.0/neuralnetwork/DL/tensorflow/parallel/LSTM.py --- .../DL/tensorflow/parallel/LSTM.py | 44 ------------------- 1 file changed, 44 deletions(-) delete mode 100644 7.0/neuralnetwork/DL/tensorflow/parallel/LSTM.py diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/LSTM.py b/7.0/neuralnetwork/DL/tensorflow/parallel/LSTM.py deleted file mode 100644 index 7319a0e3..00000000 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/LSTM.py +++ /dev/null @@ -1,44 +0,0 @@ -import tensorflow as tf -from Note.nn.layer.LSTM import LSTM -from Note.nn.layer.dense import dense -from Note.nn.parallel.optimizer import Adam -from Note.nn.Module import Module - -# Define a recurrent neural network class with LSTM layers -class lstm: - def __init__(self): - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.optimizer=Adam() - # Initialize a variable to keep track of the batch count - self.bc=tf.Variable(0, dtype=tf.float32) - - - def build(self): - # Create two LSTM layers with 50 hidden units each - self.lstm1=LSTM(50,3,return_sequence=True) # First LSTM layer that returns the full sequence - self.lstm2=LSTM(50,50) # Second LSTM layer that returns the last output only - # Create a dense layer with 10 output units - self.dense=dense(10,50) - # Store the parameters of the layers in a list - self.param=Module.param - return - - - def fp(self,data): - # Perform forward propagation on the input data - x=self.lstm1.output(data) # First LSTM layer output - x=self.lstm2.output(x) # Second LSTM layer output - output=self.dense.output(x) # Dense layer output - return output - - - def loss(self, output, labels): - # Compute the loss between the output and the labels - return self.loss_object(labels, output) - - - def opt(self,gradient): - # Perform optimization on the parameters using the gradient and the batch count - param=self.optimizer.opt(gradient,self.param,self.bc[0]) - return param From 789c865c8ac09a87d9e7cb163f8ae13a95730647 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Fri, 29 Sep 2023 21:32:53 +0800 Subject: [PATCH 214/337] Delete 7.0/neuralnetwork/DL/tensorflow/non_parallel/lstm.py --- .../DL/tensorflow/non_parallel/lstm.py | 26 ------------------- 1 file changed, 26 deletions(-) delete mode 100644 7.0/neuralnetwork/DL/tensorflow/non_parallel/lstm.py diff --git a/7.0/neuralnetwork/DL/tensorflow/non_parallel/lstm.py b/7.0/neuralnetwork/DL/tensorflow/non_parallel/lstm.py deleted file mode 100644 index 29b9b396..00000000 --- a/7.0/neuralnetwork/DL/tensorflow/non_parallel/lstm.py +++ /dev/null @@ -1,26 +0,0 @@ -import tensorflow as tf - - -class lstm: - def __init__(self,encoder): - self.model=tf.keras.Sequential([ - encoder, - tf.keras.layers.Embedding(len(encoder.get_vocabulary()), 64, mask_zero=True), - tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(64, return_sequences=True)), - tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(32)), - tf.keras.layers.Dense(64, activation='relu'), - tf.keras.layers.Dropout(0.5), - tf.keras.layers.Dense(1) - ]) - self.param=self.model.weights[1:] - self.opt=tf.keras.optimizers.Adam() - - - def fp(self,data): - output=self.model(data) - return output - - - def loss(self,output,labels): - bce=tf.keras.losses.BinaryCrossentropy(from_logits=True) - return bce(labels,output) \ No newline at end of file From 32d37427e4c602d7c6a81014505d8f60e22607ef Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Tue, 3 Oct 2023 14:22:37 +0800 Subject: [PATCH 215/337] Add files via upload --- 7.0/neuralnetwork/DL/tensorflow/parallel/cnn.py | 4 ++-- 7.0/neuralnetwork/DL/tensorflow/parallel/cnn_lmix.py | 4 ++-- 7.0/neuralnetwork/DL/tensorflow/parallel/nn.py | 6 +++--- 7.0/neuralnetwork/DL/tensorflow/parallel/nn_.py | 4 ++-- 7.0/neuralnetwork/DL/tensorflow/parallel/nn_acc.py | 6 +++--- 7.0/neuralnetwork/DL/tensorflow/parallel/nn_attenuate.py | 6 +++--- 7.0/neuralnetwork/DL/tensorflow/parallel/nn_clipping.py | 6 +++--- 7.0/neuralnetwork/DL/tensorflow/parallel/nn_device.py | 6 +++--- 8 files changed, 21 insertions(+), 21 deletions(-) diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/cnn.py b/7.0/neuralnetwork/DL/tensorflow/parallel/cnn.py index fc3182d1..56a25a86 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/cnn.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/cnn.py @@ -10,7 +10,6 @@ class cnn: def __init__(self): # Initialize the loss function and the optimizer self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.optimizer=Adam() # Initialize a variable to keep track of the batch count self.bc=tf.Variable(0,dtype=tf.float32) @@ -23,6 +22,7 @@ def build(self): # Create two dense layers with relu and linear activations self.dense1=dense(64,64*4*4,activation='relu') self.dense2=dense(10,64) + self.optimizer=Adam() # Store the parameters of the layers in a list self.param=Module.param return @@ -51,4 +51,4 @@ def loss(self,output,labels): def opt(self,gradient): # Perform optimization on the parameters using the gradient and the batch count param=self.optimizer.opt(gradient,self.param,self.bc[0]) - return param + return param \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/cnn_lmix.py b/7.0/neuralnetwork/DL/tensorflow/parallel/cnn_lmix.py index 822b03ed..b808d176 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/cnn_lmix.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/cnn_lmix.py @@ -11,7 +11,6 @@ class cnn: def __init__(self): # Initialize the loss function and the optimizer self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.optimizer=Adam() # Initialize the alpha parameter for mixup self.alpha=1.0 # Initialize a variable to keep track of the batch count @@ -26,6 +25,7 @@ def build(self): # Create two dense layers with relu and linear activations self.dense1=dense(64,64*4*4,activation='relu') self.dense2=dense(10,64) + self.optimizer=Adam() # Store the parameters of the layers in a list self.param=Module.param return @@ -59,4 +59,4 @@ def loss(self,output,labels): def opt(self,gradient): # Perform optimization on the parameters using the gradient and the batch count param=self.optimizer.opt(gradient,self.param,self.bc[0]) - return param + return param \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn.py index 75f4df3c..56e89ffb 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/nn.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/nn.py @@ -1,6 +1,6 @@ import tensorflow as tf import Note.nn.layer.dense as d -from Note.nn.parallel.optimizer import Momentum +from Note.nn.parallel.optimizer import SGD from Note.nn.layer.flatten import flatten from Note.nn.Module import Module @@ -9,13 +9,13 @@ class nn: def __init__(self): # Initialize the loss function and the optimizer self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.optimizer=Momentum(0.07,0.7) def build(self): # Create two dense layers with relu and linear activations self.layer1=d.dense(128,784,activation='relu') self.layer2=d.dense(10,128) + self.optimizer=SGD() # Store the parameters of the layers in a list self.param=Module.param return @@ -37,4 +37,4 @@ def loss(self,output,labels): def opt(self,gradient): # Perform optimization on the parameters using the gradient param=self.optimizer.opt(gradient,self.param) - return param + return param \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_.py index d115cc0c..7f7d3758 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_.py @@ -1,5 +1,5 @@ import tensorflow as tf -from Note.nn.parallel.optimizer import Momentum +from Note.nn.parallel.optimizer import SGD from Note.nn.layer.flatten import flatten # Define a neural network class @@ -16,7 +16,6 @@ def __init__(self): self.param=[self.weight1,self.weight2,self.weight3,self.bias1,self.bias2,self.bias3] # Initialize the loss function and the optimizer self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.optimizer=Momentum(0.07,0.7) def fp(self,data): @@ -25,6 +24,7 @@ def fp(self,data): layer1=tf.nn.relu(tf.matmul(data,self.weight1)+self.bias1) # First layer with relu activation layer2=tf.nn.relu(tf.matmul(layer1,self.weight2)+self.bias2) # Second layer with relu activation output=tf.matmul(layer2,self.weight3)+self.bias3 # Output layer with linear activation + self.optimizer=SGD(param=self.param) return output diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_acc.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_acc.py index 4ffcf38e..b54cc76f 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_acc.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_acc.py @@ -1,6 +1,6 @@ import tensorflow as tf import Note.nn.layer.dense as d -from Note.nn.parallel.optimizer import Momentum +from Note.nn.parallel.optimizer import SGD from Note.nn.layer.flatten import flatten from Note.nn.accuracy import sparse_categorical_accuracy from Note.nn.Module import Module @@ -10,13 +10,13 @@ class nn: def __init__(self): # Initialize the loss function and the optimizer self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.optimizer=Momentum(0.07,0.7) def build(self): # Create two dense layers with relu and linear activations self.layer1=d.dense(128,784,activation='relu') self.layer2=d.dense(10,128) + self.optimizer=SGD() # Store the parameters of the layers in a list self.param=Module.param return @@ -42,4 +42,4 @@ def accuracy(self,output,labels): #accuracy function,kernel uses it to calculate def opt(self,gradient): # Perform optimization on the parameters using the gradient param=self.optimizer.opt(gradient,self.param) - return param + return param \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_attenuate.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_attenuate.py index d81a059d..c51bbff2 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_attenuate.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_attenuate.py @@ -1,7 +1,7 @@ import tensorflow as tf from tensorflow.python.util import nest import Note.nn.layer.dense as d -from Note.nn.parallel.optimizer import Momentum +from Note.nn.parallel.optimizer import SGD from Note.nn.layer.flatten import flatten from Note.nn.accuracy import sparse_categorical_accuracy from Note.nn.Module import Module @@ -11,7 +11,6 @@ class nn: def __init__(self): # Initialize the loss function and the optimizer self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.optimizer=Momentum(0.07,0.7) # Initialize a variable to keep track of the number of optimization steps self.opt_counter=tf.Variable(tf.zeros(3,dtype=tf.float32)) @@ -20,6 +19,7 @@ def build(self): # Create two dense layers with relu and linear activations self.layer1=d.dense(128,784,activation='relu') self.layer2=d.dense(10,128) + self.optimizer=SGD() # Store the parameters of the layers in a list self.param=Module.param return @@ -55,4 +55,4 @@ def attenuate(self,gradient,oc,p): #gradient attenuation function,kernel uses i def opt(self,gradient): # Perform optimization on the parameters using the gradient param=self.optimizer.opt(gradient,self.param) # Use the optimizer to update the parameters - return param + return param \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_clipping.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_clipping.py index 6a3c8e71..6398645d 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_clipping.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_clipping.py @@ -1,6 +1,6 @@ import tensorflow as tf import Note.nn.layer.dense as d -from Note.nn.parallel.optimizer import Momentum +from Note.nn.parallel.optimizer import SGD from Note.nn.layer.flatten import flatten from Note.nn.gradient_clipping import gradient_clipping from Note.nn.Module import Module @@ -10,13 +10,13 @@ class nn: def __init__(self): # Initialize the loss function and the optimizer self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.optimizer=Momentum(0.07,0.7) def build(self): # Create two dense layers with relu and linear activations self.layer1=d.dense(128,784,activation='relu') self.layer2=d.dense(10,128) + self.optimizer=SGD() # Store the parameters of the layers in a list self.param=Module.param return @@ -45,4 +45,4 @@ def gradient(self,tape,loss): def opt(self,gradient): # Perform optimization on the parameters using the gradient param=self.optimizer.opt(gradient,self.param) - return param + return param \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_device.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_device.py index 37a3cb3b..bef89d90 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_device.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_device.py @@ -1,7 +1,7 @@ import tensorflow as tf # import TensorFlow library import Note.nn.layer.dense as d # import Note's dense layer module from Note.nn.layer.flatten import flatten # import Note's flatten layer function -from Note.nn.parallel.optimizer import Momentum # import Note's momentum optimizer module +from Note.nn.parallel.optimizer import SGD # import Note's momentum optimizer module from Note.nn.parallel.assign_device import assign_device # import the function to assign device according to the process index and the device type from Note.nn.Module import Module @@ -9,13 +9,13 @@ class nn: # A neural network class example, allocate device for multiple threads def __init__(self): # initialize the network self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) # loss object, kernel uses it to calculate loss. Here we use sparse categorical crossentropy with logits as output - self.optimizer=Momentum(0.07,0.7) # optimizer, kernel uses it to optimize. Here we use a custom momentum optimizer with learning rate 0.07 and momentum 0.7 def build(self): # build function, kernel uses it to create the network layers # Create two dense layers with relu and linear activations self.layer1=d.dense(128,784,activation='relu') # the first layer with 784 input units and 128 output units and ReLU activation self.layer2=d.dense(10,128) # the second layer with 128 input units and 10 output units and linear activation + self.optimizer=SGD() # Store the parameters of the layers in a list self.param=Module.param # parameter list of both layers, kernel uses it list for backpropagation return @@ -46,4 +46,4 @@ def opt(self,gradient,p): # optimization function, kernel uses it to optimize pa # Perform optimization on the parameters using the gradient with tf.device(assign_device(p,'GPU')): # assign the device according to the process index p param=self.optimizer.opt(gradient,self.param) # apply the Note's momentum optimizer to update the parameters using the gradient - return param + return param \ No newline at end of file From b0093fb97c56ad1deb285653a616a0179e0db323 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Tue, 3 Oct 2023 14:25:23 +0800 Subject: [PATCH 216/337] Add files via upload --- 7.0/neuralnetwork/RL/tensorflow/parallel/DDPG.py | 3 ++- 7.0/neuralnetwork/RL/tensorflow/parallel/DQN.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/7.0/neuralnetwork/RL/tensorflow/parallel/DDPG.py b/7.0/neuralnetwork/RL/tensorflow/parallel/DDPG.py index 2705adbb..f4c2b4ef 100644 --- a/7.0/neuralnetwork/RL/tensorflow/parallel/DDPG.py +++ b/7.0/neuralnetwork/RL/tensorflow/parallel/DDPG.py @@ -1,6 +1,7 @@ import tensorflow as tf # import TensorFlow library import numpy as np # import NumPy library import gym # import OpenAI Gym library +import Note.nn.parallel.optimizer as o # import Note's optimizer module class actor: # define a class for the actor network @@ -51,7 +52,7 @@ def __init__(self,hidden_dim,sigma,gamma,tau,actor_lr,critic_lr): # initialize t self.sigma=sigma # noise scale self.gamma=gamma # discount factor self.tau=tau # soft update factor - self.opt=tf.keras.optimizers.Adam() # optimizer, kernel uses it to optimize. Here we use Adam optimizer + self.optimizer=o.SGD(param=self.param) # optimizer, kernel uses it to optimize. Here we use a custom SGD optimizer def noise(self): # noise function, kernel uses it to generate exploration noise diff --git a/7.0/neuralnetwork/RL/tensorflow/parallel/DQN.py b/7.0/neuralnetwork/RL/tensorflow/parallel/DQN.py index d7756475..9fa3f591 100644 --- a/7.0/neuralnetwork/RL/tensorflow/parallel/DQN.py +++ b/7.0/neuralnetwork/RL/tensorflow/parallel/DQN.py @@ -21,7 +21,7 @@ def __init__(self,state_dim,hidden_dim,action_dim): # initialize the agent with self.nn=Qnet(state_dim,hidden_dim,action_dim) # create a Q-network for the agent self.target_q_net=Qnet(state_dim,hidden_dim,action_dim) # create a target Q-network for the agent self.param=self.nn.param # parameter list, kernel uses it list for backpropagation - self.optimizer=o.Momentum(0.07,0.7) # optimizer, kernel uses it to optimize. Here we use a custom momentum optimizer with learning rate 0.07 and momentum 0.7 + self.optimizer=o.SGD(param=self.param) # optimizer, kernel uses it to optimize. Here we use a custom SGD optimizer self.genv=[gym.make('CartPole-v0') for _ in range(5)] # create a list of 5 environments From 41c27747c197315ed0a7af00797a784388240261 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Wed, 4 Oct 2023 21:47:22 +0800 Subject: [PATCH 217/337] Update cnn_lmix.py --- 7.0/neuralnetwork/DL/tensorflow/parallel/cnn_lmix.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/cnn_lmix.py b/7.0/neuralnetwork/DL/tensorflow/parallel/cnn_lmix.py index b808d176..357f1773 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/cnn_lmix.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/cnn_lmix.py @@ -13,8 +13,6 @@ def __init__(self): self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) # Initialize the alpha parameter for mixup self.alpha=1.0 - # Initialize a variable to keep track of the batch count - self.bc=tf.Variable(0,dtype=tf.float32) def build(self): @@ -59,4 +57,4 @@ def loss(self,output,labels): def opt(self,gradient): # Perform optimization on the parameters using the gradient and the batch count param=self.optimizer.opt(gradient,self.param,self.bc[0]) - return param \ No newline at end of file + return param From 2960dd0415aeaf0e21a6f96760c375e225168ccf Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Wed, 4 Oct 2023 21:47:39 +0800 Subject: [PATCH 218/337] Update cnn.py --- 7.0/neuralnetwork/DL/tensorflow/parallel/cnn.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/cnn.py b/7.0/neuralnetwork/DL/tensorflow/parallel/cnn.py index 56a25a86..0a346e66 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/cnn.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/cnn.py @@ -10,8 +10,6 @@ class cnn: def __init__(self): # Initialize the loss function and the optimizer self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - # Initialize a variable to keep track of the batch count - self.bc=tf.Variable(0,dtype=tf.float32) def build(self): @@ -51,4 +49,4 @@ def loss(self,output,labels): def opt(self,gradient): # Perform optimization on the parameters using the gradient and the batch count param=self.optimizer.opt(gradient,self.param,self.bc[0]) - return param \ No newline at end of file + return param From e37c9f0a10efde608f9b348d2a9818edbe2cd6d4 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 5 Oct 2023 14:15:04 +0800 Subject: [PATCH 219/337] Update nn_device.py --- 7.0/neuralnetwork/DL/tensorflow/parallel/nn_device.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_device.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_device.py index bef89d90..334e10dc 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_device.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_device.py @@ -1,5 +1,5 @@ import tensorflow as tf # import TensorFlow library -import Note.nn.layer.dense as d # import Note's dense layer module +from Note.nn.layer.dense import dense # import dense layer class from Note.nn.layer.flatten import flatten # import Note's flatten layer function from Note.nn.parallel.optimizer import SGD # import Note's momentum optimizer module from Note.nn.parallel.assign_device import assign_device # import the function to assign device according to the process index and the device type @@ -13,8 +13,8 @@ def __init__(self): # initialize the network def build(self): # build function, kernel uses it to create the network layers # Create two dense layers with relu and linear activations - self.layer1=d.dense(128,784,activation='relu') # the first layer with 784 input units and 128 output units and ReLU activation - self.layer2=d.dense(10,128) # the second layer with 128 input units and 10 output units and linear activation + self.layer1=dense(128,784,activation='relu') # the first layer with 784 input units and 128 output units and ReLU activation + self.layer2=dense(10,128) # the second layer with 128 input units and 10 output units and linear activation self.optimizer=SGD() # Store the parameters of the layers in a list self.param=Module.param # parameter list of both layers, kernel uses it list for backpropagation @@ -46,4 +46,4 @@ def opt(self,gradient,p): # optimization function, kernel uses it to optimize pa # Perform optimization on the parameters using the gradient with tf.device(assign_device(p,'GPU')): # assign the device according to the process index p param=self.optimizer.opt(gradient,self.param) # apply the Note's momentum optimizer to update the parameters using the gradient - return param \ No newline at end of file + return param From 9abe22e4f163d60919434c1a6a906e35c5ab5063 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 5 Oct 2023 14:15:18 +0800 Subject: [PATCH 220/337] Update nn_clipping.py --- 7.0/neuralnetwork/DL/tensorflow/parallel/nn_clipping.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_clipping.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_clipping.py index 6398645d..1876a38e 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_clipping.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_clipping.py @@ -1,5 +1,5 @@ import tensorflow as tf -import Note.nn.layer.dense as d +from Note.nn.layer.dense import dense from Note.nn.parallel.optimizer import SGD from Note.nn.layer.flatten import flatten from Note.nn.gradient_clipping import gradient_clipping @@ -14,8 +14,8 @@ def __init__(self): def build(self): # Create two dense layers with relu and linear activations - self.layer1=d.dense(128,784,activation='relu') - self.layer2=d.dense(10,128) + self.layer1=dense(128,784,activation='relu') + self.layer2=dense(10,128) self.optimizer=SGD() # Store the parameters of the layers in a list self.param=Module.param @@ -45,4 +45,4 @@ def gradient(self,tape,loss): def opt(self,gradient): # Perform optimization on the parameters using the gradient param=self.optimizer.opt(gradient,self.param) - return param \ No newline at end of file + return param From 99a106ccc12fd212cc1b784120d20303b4ef3299 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 5 Oct 2023 14:15:40 +0800 Subject: [PATCH 221/337] Update nn_attenuate.py --- .../DL/tensorflow/parallel/nn_attenuate.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_attenuate.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_attenuate.py index c51bbff2..9f67d825 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_attenuate.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_attenuate.py @@ -1,6 +1,6 @@ import tensorflow as tf from tensorflow.python.util import nest -import Note.nn.layer.dense as d +from Note.nn.layer.dense import dense from Note.nn.parallel.optimizer import SGD from Note.nn.layer.flatten import flatten from Note.nn.accuracy import sparse_categorical_accuracy @@ -11,14 +11,12 @@ class nn: def __init__(self): # Initialize the loss function and the optimizer self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - # Initialize a variable to keep track of the number of optimization steps - self.opt_counter=tf.Variable(tf.zeros(3,dtype=tf.float32)) def build(self): # Create two dense layers with relu and linear activations - self.layer1=d.dense(128,784,activation='relu') - self.layer2=d.dense(10,128) + self.layer1=dense(128,784,activation='relu') + self.layer2=dense(10,128) self.optimizer=SGD() # Store the parameters of the layers in a list self.param=Module.param @@ -47,7 +45,7 @@ def attenuate(self,gradient,oc,p): #gradient attenuation function,kernel uses i ac=0.9**oc[0][p] #ac:attenuation coefficient gradient_flat=nest.flatten(gradient) # Flatten the gradient to a one-dimensional vector for i in range(len(gradient_flat)): #oc:optimization counter - gradient_flat[i]=ac*gradient_flat[i] #p:process number + gradient_flat[i].assign(tf.cast(ac,gradient_flat[i].dtype)*gradient_flat[i]) #p:process number gradient=nest.pack_sequence_as(gradient,gradient_flat) # Restore the gradient to its original shape return gradient @@ -55,4 +53,4 @@ def attenuate(self,gradient,oc,p): #gradient attenuation function,kernel uses i def opt(self,gradient): # Perform optimization on the parameters using the gradient param=self.optimizer.opt(gradient,self.param) # Use the optimizer to update the parameters - return param \ No newline at end of file + return param From 924a55b6d29b0414f4da0debb9794299211d2833 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 5 Oct 2023 14:15:54 +0800 Subject: [PATCH 222/337] Update nn_acc.py --- 7.0/neuralnetwork/DL/tensorflow/parallel/nn_acc.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_acc.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_acc.py index b54cc76f..698fcfd2 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_acc.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_acc.py @@ -1,5 +1,5 @@ import tensorflow as tf -import Note.nn.layer.dense as d +from Note.nn.layer.dense import dense from Note.nn.parallel.optimizer import SGD from Note.nn.layer.flatten import flatten from Note.nn.accuracy import sparse_categorical_accuracy @@ -14,8 +14,8 @@ def __init__(self): def build(self): # Create two dense layers with relu and linear activations - self.layer1=d.dense(128,784,activation='relu') - self.layer2=d.dense(10,128) + self.layer1=dense(128,784,activation='relu') + self.layer2=dense(10,128) self.optimizer=SGD() # Store the parameters of the layers in a list self.param=Module.param @@ -42,4 +42,4 @@ def accuracy(self,output,labels): #accuracy function,kernel uses it to calculate def opt(self,gradient): # Perform optimization on the parameters using the gradient param=self.optimizer.opt(gradient,self.param) - return param \ No newline at end of file + return param From 4e605e44537ddcdaa96d27280b17ac526d818157 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 5 Oct 2023 14:16:11 +0800 Subject: [PATCH 223/337] Update nn.py --- 7.0/neuralnetwork/DL/tensorflow/parallel/nn.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn.py index 56e89ffb..cea19931 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/nn.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/nn.py @@ -1,5 +1,5 @@ import tensorflow as tf -import Note.nn.layer.dense as d +from Note.nn.layer.dense import dense from Note.nn.parallel.optimizer import SGD from Note.nn.layer.flatten import flatten from Note.nn.Module import Module @@ -13,8 +13,8 @@ def __init__(self): def build(self): # Create two dense layers with relu and linear activations - self.layer1=d.dense(128,784,activation='relu') - self.layer2=d.dense(10,128) + self.layer1=dense(128,784,activation='relu') + self.layer2=dense(10,128) self.optimizer=SGD() # Store the parameters of the layers in a list self.param=Module.param @@ -37,4 +37,4 @@ def loss(self,output,labels): def opt(self,gradient): # Perform optimization on the parameters using the gradient param=self.optimizer.opt(gradient,self.param) - return param \ No newline at end of file + return param From aabb1384e788b0ed9fa78ca00aefb74403db8d98 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 5 Oct 2023 19:52:08 +0800 Subject: [PATCH 224/337] Update nn.py --- 7.0/neuralnetwork/DL/tensorflow/parallel/nn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn.py index cea19931..133c2ac1 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/nn.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/nn.py @@ -23,7 +23,7 @@ def build(self): def fp(self,data): # Perform forward propagation on the input data - data=flatten(data) + data=flatten().output(data) output1=self.layer1.output(data) output2=self.layer2.output(output1) return output2 From 4b6539fac2592486dc293e24a533bee7bb765549 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 5 Oct 2023 19:53:05 +0800 Subject: [PATCH 225/337] Update nn_.py --- 7.0/neuralnetwork/DL/tensorflow/parallel/nn_.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_.py index 7f7d3758..9e37d185 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_.py @@ -6,12 +6,12 @@ class nn: def __init__(self): # Initialize the weights and biases of three layers with random values - self.weight1=tf.Variable(tf.random.normal([784,64],dtype='float64')) - self.bias1=tf.Variable(tf.random.normal([64],dtype='float64')) - self.weight2=tf.Variable(tf.random.normal([64,64],dtype='float64')) - self.bias2=tf.Variable(tf.random.normal([64],dtype='float64')) - self.weight3=tf.Variable(tf.random.normal([64,10],dtype='float64')) - self.bias3=tf.Variable(tf.random.normal([10],dtype='float64')) + self.weight1=tf.Variable(tf.random.normal([784,64])) + self.bias1=tf.Variable(tf.random.normal([64])) + self.weight2=tf.Variable(tf.random.normal([64,64])) + self.bias2=tf.Variable(tf.random.normal([64])) + self.weight3=tf.Variable(tf.random.normal([64,10])) + self.bias3=tf.Variable(tf.random.normal([10])) # Store the parameters of the layers in a list self.param=[self.weight1,self.weight2,self.weight3,self.bias1,self.bias2,self.bias3] # Initialize the loss function and the optimizer @@ -20,7 +20,7 @@ def __init__(self): def fp(self,data): # Perform forward propagation on the input data - data=flatten(data) + data=flatten().output(data) layer1=tf.nn.relu(tf.matmul(data,self.weight1)+self.bias1) # First layer with relu activation layer2=tf.nn.relu(tf.matmul(layer1,self.weight2)+self.bias2) # Second layer with relu activation output=tf.matmul(layer2,self.weight3)+self.bias3 # Output layer with linear activation @@ -36,4 +36,4 @@ def loss(self,output,labels): def opt(self,gradient): # Perform optimization on the parameters using the gradient param=self.optimizer.opt(gradient,self.param) - return param \ No newline at end of file + return param From 4d3a5bbaab933cf006a2be87a8bdf45eb7d3702d Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 5 Oct 2023 19:53:17 +0800 Subject: [PATCH 226/337] Update nn_acc.py --- 7.0/neuralnetwork/DL/tensorflow/parallel/nn_acc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_acc.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_acc.py index 698fcfd2..bbfabd8e 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_acc.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_acc.py @@ -24,7 +24,7 @@ def build(self): def fp(self,data): # Perform forward propagation on the input data - data=flatten(data) + data=flatten().output(data) output1=self.layer1.output(data) output2=self.layer2.output(output1) return output2 From 5aca1dc5f919c99da07701c9f2928fc34ad50f19 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 5 Oct 2023 19:53:31 +0800 Subject: [PATCH 227/337] Update nn_device.py --- 7.0/neuralnetwork/DL/tensorflow/parallel/nn_device.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_device.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_device.py index 334e10dc..ba8f1c4e 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_device.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_device.py @@ -23,7 +23,7 @@ def build(self): # build function, kernel uses it to create the network layers def fp(self,data,p): # forward propagation function, kernel uses it for forward propagation with tf.device(assign_device(p,'GPU')): # assign the device according to the process index p - data=flatten(data) # flatten the data to a vector of 784 elements + data=flatten().output(data) # flatten the data to a vector of 784 elements output1=self.layer1.output(data) # pass the data through the first layer and get the output output2=self.layer2.output(output1) # pass the output of the first layer through the second layer and get the final output logits return output2 From 1ab0728b961d60daa174cbfe6a0b3afdd42f3423 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 5 Oct 2023 19:53:43 +0800 Subject: [PATCH 228/337] Update nn_clipping.py --- 7.0/neuralnetwork/DL/tensorflow/parallel/nn_clipping.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_clipping.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_clipping.py index 1876a38e..1e878d88 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_clipping.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_clipping.py @@ -24,7 +24,7 @@ def build(self): def fp(self,data): # Perform forward propagation on the input data - data=flatten(data) + data=flatten().output(data) output1=self.layer1.output(data) output2=self.layer2.output(output1) return output2 From 8ddfd149fcfdd670223b30883cd8e0f1c571cb06 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 5 Oct 2023 19:53:55 +0800 Subject: [PATCH 229/337] Update nn_attenuate.py --- 7.0/neuralnetwork/DL/tensorflow/parallel/nn_attenuate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_attenuate.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_attenuate.py index 9f67d825..629165f5 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_attenuate.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_attenuate.py @@ -25,7 +25,7 @@ def build(self): def fp(self,data): # Perform forward propagation on the input data - data=flatten(data) # Flatten the data to a one-dimensional vector + data=flatten().output(data) # Flatten the data to a one-dimensional vector output1=self.layer1.output(data) # Apply the first layer to the data and get the output output2=self.layer2.output(output1) # Apply the second layer to the output of the first layer and get the final output return output2 @@ -45,7 +45,7 @@ def attenuate(self,gradient,oc,p): #gradient attenuation function,kernel uses i ac=0.9**oc[0][p] #ac:attenuation coefficient gradient_flat=nest.flatten(gradient) # Flatten the gradient to a one-dimensional vector for i in range(len(gradient_flat)): #oc:optimization counter - gradient_flat[i].assign(tf.cast(ac,gradient_flat[i].dtype)*gradient_flat[i]) #p:process number + gradient_flat[i]=tf.cast(ac,gradient_flat[i].dtype)*gradient_flat[i] #p:process number gradient=nest.pack_sequence_as(gradient,gradient_flat) # Restore the gradient to its original shape return gradient From 58c75e4b1d50dbc8579f0f83415d8ae3df68ab1d Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Mon, 16 Oct 2023 14:42:24 +0800 Subject: [PATCH 230/337] Update cnn_lmix.py --- 7.0/neuralnetwork/DL/tensorflow/parallel/cnn_lmix.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/cnn_lmix.py b/7.0/neuralnetwork/DL/tensorflow/parallel/cnn_lmix.py index 357f1773..56603986 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/cnn_lmix.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/cnn_lmix.py @@ -20,6 +20,7 @@ def build(self): self.conv1=conv2d(32,[3,3],3,strides=(1,1),padding='SAME',activation='relu') self.conv2=conv2d(64,[3,3],32,strides=(2,2),padding='SAME',activation='relu') self.conv3=conv2d(64,[3,3],64,strides=(1,1),padding='SAME',activation='relu') + self.flatten=flatten() # Create two dense layers with relu and linear activations self.dense1=dense(64,64*4*4,activation='relu') self.dense2=dense(10,64) @@ -37,14 +38,14 @@ def data_func(self,data_batch,labels_batch): def fp(self,data): # Perform forward propagation on the input data x=self.conv1.output(data) # First convolutional layer - x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # First max pooling layer + x=tf.nn.max_pool2d(x,ksize=(2,2)) # First max pooling layer x=self.conv2.output(x) # Second convolutional layer x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # Second max pooling layer x=self.conv3.output(x) # Third convolutional layer - x=flatten(x) # Flatten the output to a vector - x=tf.nn.relu(tf.matmul(x,self.dense1.weight)+self.dense1.bias) # First dense layer with relu activation + x=self.flatten.output(x) # Flatten the output to a vector + x=self.dense1.output(x) # First dense layer with relu activation x=tf.nn.dropout(x,rate=0.5) # Apply dropout to prevent overfitting - output=tf.matmul(x,self.dense2.weight)+self.dense2.bias # Output layer with linear activation + output=self.dense2.output(x) # Output layer with linear activation output=tf.nn.dropout(output,rate=0.5) # Apply dropout to prevent overfitting return output From 40c21118405affbe2e30bd6a6e87792888ee7253 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Mon, 16 Oct 2023 14:42:37 +0800 Subject: [PATCH 231/337] Update cnn.py --- 7.0/neuralnetwork/DL/tensorflow/parallel/cnn.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/cnn.py b/7.0/neuralnetwork/DL/tensorflow/parallel/cnn.py index 0a346e66..40f658c1 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/cnn.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/cnn.py @@ -17,6 +17,7 @@ def build(self): self.conv1=conv2d(32,[3,3],3,strides=(1,1),padding='SAME',activation='relu') self.conv2=conv2d(64,[3,3],32,strides=(2,2),padding='SAME',activation='relu') self.conv3=conv2d(64,[3,3],64,strides=(1,1),padding='SAME',activation='relu') + self.flatten=flatten() # Create two dense layers with relu and linear activations self.dense1=dense(64,64*4*4,activation='relu') self.dense2=dense(10,64) @@ -33,10 +34,10 @@ def fp(self,data): x=self.conv2.output(x) # Second convolutional layer x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # Second max pooling layer x=self.conv3.output(x) # Third convolutional layer - x=flatten(x) # Flatten the output to a vector - x=tf.nn.relu(tf.matmul(x,self.dense1.weight)+self.dense1.bias) # First dense layer with relu activation + x=self.flatten.output(x) # Flatten the output to a vector + x=self.dense1.output(x) # First dense layer with relu activation x=tf.nn.dropout(x,rate=0.5) # Apply dropout to prevent overfitting - output=tf.matmul(x,self.dense2.weight)+self.dense2.bias # Output layer with linear activation + output=self.dense2.output(x) # Output layer with linear activation output=tf.nn.dropout(output,rate=0.5) # Apply dropout to prevent overfitting return output From fd34bc29c817b17b1cfaa5e2b462e436905f73ff Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Tue, 17 Oct 2023 14:22:35 +0800 Subject: [PATCH 232/337] Update nn_device.py --- 7.0/neuralnetwork/DL/tensorflow/parallel/nn_device.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_device.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_device.py index ba8f1c4e..1f24083b 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_device.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_device.py @@ -15,6 +15,7 @@ def build(self): # build function, kernel uses it to create the network layers # Create two dense layers with relu and linear activations self.layer1=dense(128,784,activation='relu') # the first layer with 784 input units and 128 output units and ReLU activation self.layer2=dense(10,128) # the second layer with 128 input units and 10 output units and linear activation + self.flatten=flatten() self.optimizer=SGD() # Store the parameters of the layers in a list self.param=Module.param # parameter list of both layers, kernel uses it list for backpropagation @@ -23,7 +24,7 @@ def build(self): # build function, kernel uses it to create the network layers def fp(self,data,p): # forward propagation function, kernel uses it for forward propagation with tf.device(assign_device(p,'GPU')): # assign the device according to the process index p - data=flatten().output(data) # flatten the data to a vector of 784 elements + data=self.flatten.output(data) # flatten the data to a vector of 784 elements output1=self.layer1.output(data) # pass the data through the first layer and get the output output2=self.layer2.output(output1) # pass the output of the first layer through the second layer and get the final output logits return output2 From 3400816aea02b10c8144e57ff2e60d565141063c Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Tue, 17 Oct 2023 14:22:46 +0800 Subject: [PATCH 233/337] Update nn_clipping.py --- 7.0/neuralnetwork/DL/tensorflow/parallel/nn_clipping.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_clipping.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_clipping.py index 1e878d88..46aa5eb9 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_clipping.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_clipping.py @@ -16,6 +16,7 @@ def build(self): # Create two dense layers with relu and linear activations self.layer1=dense(128,784,activation='relu') self.layer2=dense(10,128) + self.flatten=flatten() self.optimizer=SGD() # Store the parameters of the layers in a list self.param=Module.param @@ -24,7 +25,7 @@ def build(self): def fp(self,data): # Perform forward propagation on the input data - data=flatten().output(data) + data=self.flatten.output(data) output1=self.layer1.output(data) output2=self.layer2.output(output1) return output2 From 37ded7988581e93320fe1e146abb47f225372ee5 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Tue, 17 Oct 2023 14:22:56 +0800 Subject: [PATCH 234/337] Update nn_attenuate.py --- 7.0/neuralnetwork/DL/tensorflow/parallel/nn_attenuate.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_attenuate.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_attenuate.py index 629165f5..5f67d8b1 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_attenuate.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_attenuate.py @@ -17,6 +17,7 @@ def build(self): # Create two dense layers with relu and linear activations self.layer1=dense(128,784,activation='relu') self.layer2=dense(10,128) + self.flatten=flatten() self.optimizer=SGD() # Store the parameters of the layers in a list self.param=Module.param @@ -25,7 +26,7 @@ def build(self): def fp(self,data): # Perform forward propagation on the input data - data=flatten().output(data) # Flatten the data to a one-dimensional vector + data=self.flatten.output(data) # Flatten the data to a one-dimensional vector output1=self.layer1.output(data) # Apply the first layer to the data and get the output output2=self.layer2.output(output1) # Apply the second layer to the output of the first layer and get the final output return output2 From 4e852369cb0d146d65422cc9fe8ab768a51b36a7 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Tue, 17 Oct 2023 14:23:06 +0800 Subject: [PATCH 235/337] Update nn_acc.py --- 7.0/neuralnetwork/DL/tensorflow/parallel/nn_acc.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_acc.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_acc.py index bbfabd8e..51895f99 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_acc.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_acc.py @@ -16,6 +16,7 @@ def build(self): # Create two dense layers with relu and linear activations self.layer1=dense(128,784,activation='relu') self.layer2=dense(10,128) + self.flatten=flatten() self.optimizer=SGD() # Store the parameters of the layers in a list self.param=Module.param @@ -24,7 +25,7 @@ def build(self): def fp(self,data): # Perform forward propagation on the input data - data=flatten().output(data) + data=self.flatten.output(data) output1=self.layer1.output(data) output2=self.layer2.output(output1) return output2 From a18c9531c398ba99eb23c03fe63d09ccd453f488 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Tue, 17 Oct 2023 14:23:18 +0800 Subject: [PATCH 236/337] Update nn.py --- 7.0/neuralnetwork/DL/tensorflow/parallel/nn.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn.py index 133c2ac1..87f71817 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/nn.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/nn.py @@ -15,6 +15,7 @@ def build(self): # Create two dense layers with relu and linear activations self.layer1=dense(128,784,activation='relu') self.layer2=dense(10,128) + self.flatten=flatten() self.optimizer=SGD() # Store the parameters of the layers in a list self.param=Module.param @@ -23,7 +24,7 @@ def build(self): def fp(self,data): # Perform forward propagation on the input data - data=flatten().output(data) + data=self.flatten.output(data) output1=self.layer1.output(data) output2=self.layer2.output(output1) return output2 From 0028cc1362488423f9b5f94daa49c13ee4870569 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Tue, 17 Oct 2023 14:34:40 +0800 Subject: [PATCH 237/337] Update cnn_lmix.py --- 7.0/neuralnetwork/DL/tensorflow/parallel/cnn_lmix.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/cnn_lmix.py b/7.0/neuralnetwork/DL/tensorflow/parallel/cnn_lmix.py index 56603986..f594aa5f 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/cnn_lmix.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/cnn_lmix.py @@ -22,7 +22,7 @@ def build(self): self.conv3=conv2d(64,[3,3],64,strides=(1,1),padding='SAME',activation='relu') self.flatten=flatten() # Create two dense layers with relu and linear activations - self.dense1=dense(64,64*4*4,activation='relu') + self.dense1=dense(64,64,activation='relu') self.dense2=dense(10,64) self.optimizer=Adam() # Store the parameters of the layers in a list From 2a7009d8d58c926935ac95da8d006b0579d14e6f Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Tue, 17 Oct 2023 14:34:52 +0800 Subject: [PATCH 238/337] Update cnn.py --- 7.0/neuralnetwork/DL/tensorflow/parallel/cnn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/cnn.py b/7.0/neuralnetwork/DL/tensorflow/parallel/cnn.py index 40f658c1..ff441b77 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/cnn.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/cnn.py @@ -19,7 +19,7 @@ def build(self): self.conv3=conv2d(64,[3,3],64,strides=(1,1),padding='SAME',activation='relu') self.flatten=flatten() # Create two dense layers with relu and linear activations - self.dense1=dense(64,64*4*4,activation='relu') + self.dense1=dense(64,64,activation='relu') self.dense2=dense(10,64) self.optimizer=Adam() # Store the parameters of the layers in a list From e2a13c76b1a6261141d2a9e7d0da7afc4b2bf7aa Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Tue, 17 Oct 2023 14:40:46 +0800 Subject: [PATCH 239/337] Update cnn_lmix.py --- 7.0/neuralnetwork/DL/tensorflow/parallel/cnn_lmix.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/cnn_lmix.py b/7.0/neuralnetwork/DL/tensorflow/parallel/cnn_lmix.py index f594aa5f..c7ef9546 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/cnn_lmix.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/cnn_lmix.py @@ -38,15 +38,13 @@ def data_func(self,data_batch,labels_batch): def fp(self,data): # Perform forward propagation on the input data x=self.conv1.output(data) # First convolutional layer - x=tf.nn.max_pool2d(x,ksize=(2,2)) # First max pooling layer + x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # First max pooling layer x=self.conv2.output(x) # Second convolutional layer x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # Second max pooling layer x=self.conv3.output(x) # Third convolutional layer x=self.flatten.output(x) # Flatten the output to a vector x=self.dense1.output(x) # First dense layer with relu activation - x=tf.nn.dropout(x,rate=0.5) # Apply dropout to prevent overfitting output=self.dense2.output(x) # Output layer with linear activation - output=tf.nn.dropout(output,rate=0.5) # Apply dropout to prevent overfitting return output From 37aa7d915b2621c85fa4847780510a866c27c875 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Tue, 17 Oct 2023 14:41:08 +0800 Subject: [PATCH 240/337] Update cnn.py --- 7.0/neuralnetwork/DL/tensorflow/parallel/cnn.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/cnn.py b/7.0/neuralnetwork/DL/tensorflow/parallel/cnn.py index ff441b77..8e130ccd 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/cnn.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/cnn.py @@ -30,15 +30,13 @@ def build(self): def fp(self,data): # Perform forward propagation on the input data x=self.conv1.output(data) # First convolutional layer - x=tf.nn.max_pool2d(x,ksize=(2,2)) # First max pooling layer + x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # First max pooling layer x=self.conv2.output(x) # Second convolutional layer x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # Second max pooling layer x=self.conv3.output(x) # Third convolutional layer x=self.flatten.output(x) # Flatten the output to a vector x=self.dense1.output(x) # First dense layer with relu activation - x=tf.nn.dropout(x,rate=0.5) # Apply dropout to prevent overfitting output=self.dense2.output(x) # Output layer with linear activation - output=tf.nn.dropout(output,rate=0.5) # Apply dropout to prevent overfitting return output From 22358bb405ea143ca677f15a35c5ed7fe5487ba9 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Tue, 17 Oct 2023 14:42:16 +0800 Subject: [PATCH 241/337] Add files via upload --- .../DL/tensorflow/non_parallel/layer/cnn.py | 48 +++++++++++++++++++ .../DL/tensorflow/non_parallel/layer/nn.py | 38 +++++++++++++++ 2 files changed, 86 insertions(+) create mode 100644 7.0/neuralnetwork/DL/tensorflow/non_parallel/layer/cnn.py create mode 100644 7.0/neuralnetwork/DL/tensorflow/non_parallel/layer/nn.py diff --git a/7.0/neuralnetwork/DL/tensorflow/non_parallel/layer/cnn.py b/7.0/neuralnetwork/DL/tensorflow/non_parallel/layer/cnn.py new file mode 100644 index 00000000..0ee576cb --- /dev/null +++ b/7.0/neuralnetwork/DL/tensorflow/non_parallel/layer/cnn.py @@ -0,0 +1,48 @@ +import tensorflow as tf +from Note.nn.layer.conv2d import conv2d +from Note.nn.layer.dense import dense +from Note.nn.layer.flatten import flatten +from Note.nn.Module import Module + +""" +This is an example of using the Note layer module. +""" + +# Define a convolutional neural network class +class cnn: + def __init__(self): + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.opt=tf.keras.optimizers.Adam() + + + def build(self): + # Create three convolutional layers with relu activations + self.conv1=conv2d(32,[3,3],strides=(1,1),padding='SAME',activation='relu') + self.conv2=conv2d(64,[3,3],strides=(2,2),padding='SAME',activation='relu') + self.conv3=conv2d(64,[3,3],strides=(1,1),padding='SAME',activation='relu') + self.flatten=flatten() + # Create two dense layers with relu and linear activations + self.dense1=dense(64,activation='relu') + self.dense2=dense(10) + # Store the parameters of the layers in a list + self.param=Module.param + return + + + def fp(self,data): + # Perform forward propagation on the input data + x=self.conv1.output(data) # First convolutional layer + x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # First max pooling layer + x=self.conv2.output(x) # Second convolutional layer + x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # Second max pooling layer + x=self.conv3.output(x) # Third convolutional layer + x=self.flatten.output(x) # Flatten the output to a vector + x=self.dense1.output(x) # First dense layer with relu activation + output=self.dense2.output(x) # Output layer with linear activation + return output + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/non_parallel/layer/nn.py b/7.0/neuralnetwork/DL/tensorflow/non_parallel/layer/nn.py new file mode 100644 index 00000000..ae9ff128 --- /dev/null +++ b/7.0/neuralnetwork/DL/tensorflow/non_parallel/layer/nn.py @@ -0,0 +1,38 @@ +import tensorflow as tf +from Note.nn.layer.dense import dense +from Note.nn.layer.flatten import flatten +from Note.nn.Module import Module + +""" +This is an example of using the Note layer module. +""" + +# Define a neural network class +class nn: + def __init__(self): + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.opt=tf.keras.optimizers.Adam() #optimizer,kernel uses it to optimize. + + + def build(self): + # Create two dense layers with relu and linear activations + self.layer1=dense(128,activation='relu') + self.layer2=dense(10) + self.flatten=flatten() + # Store the parameters of the layers in a list + self.param=Module.param + return + + + def fp(self,data): + # Perform forward propagation on the input data + data=self.flatten.output(data) + output1=self.layer1.output(data) + output2=self.layer2.output(output1) + return output2 + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) \ No newline at end of file From 259b172bd1ef5193e900ab944706ce535261aea4 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Thu, 2 Nov 2023 15:51:04 +0800 Subject: [PATCH 242/337] Update nn_attenuate.py --- 7.0/neuralnetwork/DL/tensorflow/parallel/nn_attenuate.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_attenuate.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_attenuate.py index 5f67d8b1..9f71ddd0 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_attenuate.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_attenuate.py @@ -41,11 +41,11 @@ def accuracy(self,output,labels): #accuracy function,kernel uses it to calculate return sparse_categorical_accuracy(labels,output) - def attenuate(self,gradient,oc,p): #gradient attenuation function,kernel uses it to calculate attenuation coefficient. + def attenuate(self,gradient,p): #gradient attenuation function,kernel uses it to calculate attenuation coefficient. # Apply an exponential decay to the gradient based on the optimization counter - ac=0.9**oc[0][p] #ac:attenuation coefficient + ac=0.9**self.opt_counter[0][p] #ac:attenuation coefficient gradient_flat=nest.flatten(gradient) # Flatten the gradient to a one-dimensional vector - for i in range(len(gradient_flat)): #oc:optimization counter + for i in range(len(gradient_flat)): #self.opt_counter:optimization counter gradient_flat[i]=tf.cast(ac,gradient_flat[i].dtype)*gradient_flat[i] #p:process number gradient=nest.pack_sequence_as(gradient,gradient_flat) # Restore the gradient to its original shape return gradient From 75bd2f15743b1def08a73e38a9554a070f613090 Mon Sep 17 00:00:00 2001 From: NoteDancing <63648431+NoteDancing@users.noreply.github.com> Date: Tue, 28 Nov 2023 22:53:41 +0800 Subject: [PATCH 243/337] Delete 7.0/neuralnetwork/DL/tensorflow/parallel/nn_.py --- .../DL/tensorflow/parallel/nn_.py | 39 ------------------- 1 file changed, 39 deletions(-) delete mode 100644 7.0/neuralnetwork/DL/tensorflow/parallel/nn_.py diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_.py deleted file mode 100644 index 9e37d185..00000000 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_.py +++ /dev/null @@ -1,39 +0,0 @@ -import tensorflow as tf -from Note.nn.parallel.optimizer import SGD -from Note.nn.layer.flatten import flatten - -# Define a neural network class -class nn: - def __init__(self): - # Initialize the weights and biases of three layers with random values - self.weight1=tf.Variable(tf.random.normal([784,64])) - self.bias1=tf.Variable(tf.random.normal([64])) - self.weight2=tf.Variable(tf.random.normal([64,64])) - self.bias2=tf.Variable(tf.random.normal([64])) - self.weight3=tf.Variable(tf.random.normal([64,10])) - self.bias3=tf.Variable(tf.random.normal([10])) - # Store the parameters of the layers in a list - self.param=[self.weight1,self.weight2,self.weight3,self.bias1,self.bias2,self.bias3] - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - - - def fp(self,data): - # Perform forward propagation on the input data - data=flatten().output(data) - layer1=tf.nn.relu(tf.matmul(data,self.weight1)+self.bias1) # First layer with relu activation - layer2=tf.nn.relu(tf.matmul(layer1,self.weight2)+self.bias2) # Second layer with relu activation - output=tf.matmul(layer2,self.weight3)+self.bias3 # Output layer with linear activation - self.optimizer=SGD(param=self.param) - return output - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) - - - def opt(self,gradient): - # Perform optimization on the parameters using the gradient - param=self.optimizer.opt(gradient,self.param) - return param From 89167fc4f4bf66f3b2059f4d9c8bcc908fcc798a Mon Sep 17 00:00:00 2001 From: NoteDance <63648431+NoteDance@users.noreply.github.com> Date: Sun, 24 Dec 2023 19:13:07 +0800 Subject: [PATCH 244/337] Delete 7.0/neuralnetwork/DL/tensorflow/parallel/cnn_lmix.py --- .../DL/tensorflow/parallel/cnn_lmix.py | 59 ------------------- 1 file changed, 59 deletions(-) delete mode 100644 7.0/neuralnetwork/DL/tensorflow/parallel/cnn_lmix.py diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/cnn_lmix.py b/7.0/neuralnetwork/DL/tensorflow/parallel/cnn_lmix.py deleted file mode 100644 index c7ef9546..00000000 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/cnn_lmix.py +++ /dev/null @@ -1,59 +0,0 @@ -import tensorflow as tf -from Note.nn.layer.conv2d import conv2d -from Note.nn.layer.dense import dense -from Note.nn.layer.flatten import flatten -from Note.nn.parallel.optimizer import Adam -from Note.nn.layer.LMix import lmix -from Note.nn.Module import Module - -# Define a convolutional neural network class with mixup augmentation -class cnn: - def __init__(self): - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - # Initialize the alpha parameter for mixup - self.alpha=1.0 - - - def build(self): - # Create three convolutional layers with relu activations - self.conv1=conv2d(32,[3,3],3,strides=(1,1),padding='SAME',activation='relu') - self.conv2=conv2d(64,[3,3],32,strides=(2,2),padding='SAME',activation='relu') - self.conv3=conv2d(64,[3,3],64,strides=(1,1),padding='SAME',activation='relu') - self.flatten=flatten() - # Create two dense layers with relu and linear activations - self.dense1=dense(64,64,activation='relu') - self.dense2=dense(10,64) - self.optimizer=Adam() - # Store the parameters of the layers in a list - self.param=Module.param - return - - - def data_func(self,data_batch,labels_batch): - # Apply mixup augmentation on the input data and labels - return lmix(data_batch,labels_batch,self.alpha) - - - def fp(self,data): - # Perform forward propagation on the input data - x=self.conv1.output(data) # First convolutional layer - x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # First max pooling layer - x=self.conv2.output(x) # Second convolutional layer - x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # Second max pooling layer - x=self.conv3.output(x) # Third convolutional layer - x=self.flatten.output(x) # Flatten the output to a vector - x=self.dense1.output(x) # First dense layer with relu activation - output=self.dense2.output(x) # Output layer with linear activation - return output - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) - - - def opt(self,gradient): - # Perform optimization on the parameters using the gradient and the batch count - param=self.optimizer.opt(gradient,self.param,self.bc[0]) - return param From 72b592ef45bbf8c4018aba1db9c55f84320aeb60 Mon Sep 17 00:00:00 2001 From: NoteDance <63648431+NoteDance@users.noreply.github.com> Date: Fri, 19 Jan 2024 21:44:08 +0800 Subject: [PATCH 245/337] Add files via upload --- .../RL/tensorflow/non_parallel/PPO.py | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py diff --git a/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py b/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py new file mode 100644 index 00000000..1297b4d0 --- /dev/null +++ b/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py @@ -0,0 +1,75 @@ +import tensorflow as tf +import gym + + +class actor: + def __init__(self,state_dim,hidden_dim,action_dim): + self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) + self.bias1=tf.Variable(tf.random.normal([hidden_dim])) + self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) + self.bias2=tf.Variable(tf.random.normal([action_dim])) + self.param=[self.weight1,self.bias1,self.weight2,self.bias2] + + def fp(self,x): + x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) + return tf.nn.softmax(tf.matmul(x,self.weight2)+self.bias2) + + +class critic: + def __init__(self,state_dim,hidden_dim): + self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) + self.bias1=tf.Variable(tf.random.normal([hidden_dim])) + self.weight2=tf.Variable(tf.random.normal([hidden_dim,1])) + self.bias2=tf.Variable(tf.random.normal([1])) + self.param=[self.weight1,self.bias1,self.weight2,self.bias2] + + def fp(self,x): + x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) + return tf.matmul(x,self.weight2)+self.bias2 + + +class PPO: + def __init__(self,state_dim,hidden_dim,action_dim,clip_eps): + self.nn=actor(state_dim,hidden_dim,action_dim) + self.critic=critic(state_dim,hidden_dim) + self.actor_old=actor(state_dim,hidden_dim,action_dim) + self.clip_eps=clip_eps + self.param=[self.actor.param,self.critic.param] + self.opt=tf.keras.optimizers.Adam() + self.genv=gym.make('CartPole-v0') + + + def env(self,a=None,initial=None): + if initial==True: + state=self.genv.reset(seed=0) + return state + else: + next_state,reward,done,_=self.genv.step(a) + return next_state,reward,done + + + def loss(self,s,a,next_s,r,d): + a=tf.expand_dims(a,axis=1) + raito=tf.gather(self.actor.fp(s),a,axis=1,batch_dims=1)/tf.gather(self.actor_old.fp(s),a,axis=1,batch_dims=1) + value=tf.gather(self.critic.fp(s),a,axis=1,batch_dims=1) + value_tar=r+tf.gather(self.critic.fp(next_s),a,axis=1,batch_dims=1) + TD=r+0.98*value_tar*(1-d)-value + clip=tf.clip_by_value(raito,clip_value_min=1-self.clip_eps,clip_value_max=1+self.clip_eps)*TD + return [tf.reduce_mean(clip),tf.reduce_mean((TD)**2)] + + + def gradient(self,tape,loss): + actor_gradient=tape.gradient(loss[0],self.param[0]) + critic_gradient=tape.gradient(loss[1],self.param[1]) + return [actor_gradient,critic_gradient] + + + def opt(self,gradient): + self.opt.opt(zip(gradient[0],self.param[0])) + self.opt.opt(zip(gradient[1],self.param[1])) + return + + + def update_param(self): + self.actor_old.param=self.actor.param.copy() + return \ No newline at end of file From 418e64c1d2614e6281c7478ef2ea0613b107530a Mon Sep 17 00:00:00 2001 From: NoteDance <63648431+NoteDance@users.noreply.github.com> Date: Fri, 19 Jan 2024 21:44:30 +0800 Subject: [PATCH 246/337] Add files via upload --- .../RL/tensorflow/parallel/PPO.py | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py diff --git a/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py b/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py new file mode 100644 index 00000000..fbb47795 --- /dev/null +++ b/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py @@ -0,0 +1,76 @@ +import tensorflow as tf +import gym +from Note.nn.parallel.optimizer import SGD + + +class actor: + def __init__(self,state_dim,hidden_dim,action_dim): + self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) + self.bias1=tf.Variable(tf.random.normal([hidden_dim])) + self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) + self.bias2=tf.Variable(tf.random.normal([action_dim])) + self.param=[self.weight1,self.bias1,self.weight2,self.bias2] + + def fp(self,x): + x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) + return tf.nn.softmax(tf.matmul(x,self.weight2)+self.bias2) + + +class critic: + def __init__(self,state_dim,hidden_dim): + self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) + self.bias1=tf.Variable(tf.random.normal([hidden_dim])) + self.weight2=tf.Variable(tf.random.normal([hidden_dim,1])) + self.bias2=tf.Variable(tf.random.normal([1])) + self.param=[self.weight1,self.bias1,self.weight2,self.bias2] + + def fp(self,x): + x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) + return tf.matmul(x,self.weight2)+self.bias2 + + +class PPO: + def __init__(self,state_dim,hidden_dim,action_dim,clip_eps): + self.nn=actor(state_dim,hidden_dim,action_dim) + self.critic=critic(state_dim,hidden_dim) + self.actor_old=actor(state_dim,hidden_dim,action_dim) + self.clip_eps=clip_eps + self.param=[self.actor.param,self.critic.param] + self.opt=SGD(param=self.param) + self.genv=[gym.make('CartPole-v0') for _ in range(5)] + + + def env(self,a=None,initial=None): + if initial==True: + state=self.genv.reset(seed=0) + return state + else: + next_state,reward,done,_=self.genv.step(a) + return next_state,reward,done + + + def loss(self,s,a,next_s,r,d): + a=tf.expand_dims(a,axis=1) + raito=tf.gather(self.actor.fp(s),a,axis=1,batch_dims=1)/tf.gather(self.actor_old.fp(s),a,axis=1,batch_dims=1) + value=tf.gather(self.critic.fp(s),a,axis=1,batch_dims=1) + value_tar=r+tf.gather(self.critic.fp(next_s),a,axis=1,batch_dims=1) + TD=r+0.98*value_tar*(1-d)-value + clip=tf.clip_by_value(raito,clip_value_min=1-self.clip_eps,clip_value_max=1+self.clip_eps)*TD + return [tf.reduce_mean(clip),tf.reduce_mean((TD)**2)] + + + def gradient(self,tape,loss): + actor_gradient=tape.gradient(loss[0],self.param[0]) + critic_gradient=tape.gradient(loss[1],self.param[1]) + return [actor_gradient,critic_gradient] + + + def opt(self,gradient): + self.opt.opt(zip(gradient[0],self.param[0])) + self.opt.opt(zip(gradient[1],self.param[1])) + return + + + def update_param(self): + self.actor_old.param=self.actor.param.copy() + return \ No newline at end of file From d6ca0ce3a2a3c99e5cd1376d5d24e90f8a62e950 Mon Sep 17 00:00:00 2001 From: NoteDance <63648431+NoteDance@users.noreply.github.com> Date: Fri, 19 Jan 2024 21:49:31 +0800 Subject: [PATCH 247/337] Update PPO.py --- 7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py b/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py index 1297b4d0..eb0990df 100644 --- a/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py +++ b/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py @@ -65,11 +65,7 @@ def gradient(self,tape,loss): def opt(self,gradient): - self.opt.opt(zip(gradient[0],self.param[0])) - self.opt.opt(zip(gradient[1],self.param[1])) - return - - - def update_param(self): + self.opt.apply_gradients(zip(gradient[0],self.param[0])) + self.opt.apply_gradients(zip(gradient[1],self.param[1])) self.actor_old.param=self.actor.param.copy() - return \ No newline at end of file + return From edf62e3efa6f8ac1f254f64baf9aaf25e35106b1 Mon Sep 17 00:00:00 2001 From: NoteDance <63648431+NoteDance@users.noreply.github.com> Date: Fri, 19 Jan 2024 21:50:19 +0800 Subject: [PATCH 248/337] Update PPO.py --- 7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py b/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py index fbb47795..4b26ad4b 100644 --- a/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py +++ b/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py @@ -66,11 +66,7 @@ def gradient(self,tape,loss): def opt(self,gradient): - self.opt.opt(zip(gradient[0],self.param[0])) - self.opt.opt(zip(gradient[1],self.param[1])) - return - - - def update_param(self): + self.opt.opt(gradient[0],self.param[0]) + self.opt.opt(gradient[1],self.param[1]) self.actor_old.param=self.actor.param.copy() - return \ No newline at end of file + return From 0589c3f3c159d176a4f49105cb674128308d9f0f Mon Sep 17 00:00:00 2001 From: NoteDance <63648431+NoteDance@users.noreply.github.com> Date: Fri, 19 Jan 2024 22:10:20 +0800 Subject: [PATCH 249/337] Update PPO.py --- 7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py b/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py index 4b26ad4b..c0319b11 100644 --- a/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py +++ b/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py @@ -17,11 +17,11 @@ def fp(self,x): class critic: - def __init__(self,state_dim,hidden_dim): + def __init__(self,state_dim,hidden_dim,action_dim): self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) self.bias1=tf.Variable(tf.random.normal([hidden_dim])) - self.weight2=tf.Variable(tf.random.normal([hidden_dim,1])) - self.bias2=tf.Variable(tf.random.normal([1])) + self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) + self.bias2=tf.Variable(tf.random.normal([action_dim])) self.param=[self.weight1,self.bias1,self.weight2,self.bias2] def fp(self,x): @@ -32,7 +32,7 @@ def fp(self,x): class PPO: def __init__(self,state_dim,hidden_dim,action_dim,clip_eps): self.nn=actor(state_dim,hidden_dim,action_dim) - self.critic=critic(state_dim,hidden_dim) + self.critic=critic(state_dim,hidden_dim,action_dim) self.actor_old=actor(state_dim,hidden_dim,action_dim) self.clip_eps=clip_eps self.param=[self.actor.param,self.critic.param] From a6044ebdc2f82fee8a4dff21d5dbbe955fa04a3a Mon Sep 17 00:00:00 2001 From: NoteDance <63648431+NoteDance@users.noreply.github.com> Date: Fri, 19 Jan 2024 22:10:34 +0800 Subject: [PATCH 250/337] Update PPO.py --- 7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py b/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py index eb0990df..4b27e686 100644 --- a/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py +++ b/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py @@ -16,11 +16,11 @@ def fp(self,x): class critic: - def __init__(self,state_dim,hidden_dim): + def __init__(self,state_dim,hidden_dim,action_dim): self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) self.bias1=tf.Variable(tf.random.normal([hidden_dim])) - self.weight2=tf.Variable(tf.random.normal([hidden_dim,1])) - self.bias2=tf.Variable(tf.random.normal([1])) + self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) + self.bias2=tf.Variable(tf.random.normal([action_dim])) self.param=[self.weight1,self.bias1,self.weight2,self.bias2] def fp(self,x): @@ -31,7 +31,7 @@ def fp(self,x): class PPO: def __init__(self,state_dim,hidden_dim,action_dim,clip_eps): self.nn=actor(state_dim,hidden_dim,action_dim) - self.critic=critic(state_dim,hidden_dim) + self.critic=critic(state_dim,hidden_dim,action_dim) self.actor_old=actor(state_dim,hidden_dim,action_dim) self.clip_eps=clip_eps self.param=[self.actor.param,self.critic.param] From 81893b93e3111a3c5d78b9cef5a6a52d7f93ea92 Mon Sep 17 00:00:00 2001 From: NoteDance <63648431+NoteDance@users.noreply.github.com> Date: Fri, 19 Jan 2024 22:14:25 +0800 Subject: [PATCH 251/337] Update PPO.py --- 7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py b/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py index 4b27e686..71038b36 100644 --- a/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py +++ b/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py @@ -16,11 +16,11 @@ def fp(self,x): class critic: - def __init__(self,state_dim,hidden_dim,action_dim): + def __init__(self,state_dim,hidden_dim): self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) self.bias1=tf.Variable(tf.random.normal([hidden_dim])) - self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) - self.bias2=tf.Variable(tf.random.normal([action_dim])) + self.weight2=tf.Variable(tf.random.normal([hidden_dim,1])) + self.bias2=tf.Variable(tf.random.normal([1])) self.param=[self.weight1,self.bias1,self.weight2,self.bias2] def fp(self,x): @@ -31,7 +31,7 @@ def fp(self,x): class PPO: def __init__(self,state_dim,hidden_dim,action_dim,clip_eps): self.nn=actor(state_dim,hidden_dim,action_dim) - self.critic=critic(state_dim,hidden_dim,action_dim) + self.critic=critic(state_dim,hidden_dim) self.actor_old=actor(state_dim,hidden_dim,action_dim) self.clip_eps=clip_eps self.param=[self.actor.param,self.critic.param] @@ -51,8 +51,8 @@ def env(self,a=None,initial=None): def loss(self,s,a,next_s,r,d): a=tf.expand_dims(a,axis=1) raito=tf.gather(self.actor.fp(s),a,axis=1,batch_dims=1)/tf.gather(self.actor_old.fp(s),a,axis=1,batch_dims=1) - value=tf.gather(self.critic.fp(s),a,axis=1,batch_dims=1) - value_tar=r+tf.gather(self.critic.fp(next_s),a,axis=1,batch_dims=1) + value=self.critic.fp(s) + value_tar=r+self.critic.fp(next_s) TD=r+0.98*value_tar*(1-d)-value clip=tf.clip_by_value(raito,clip_value_min=1-self.clip_eps,clip_value_max=1+self.clip_eps)*TD return [tf.reduce_mean(clip),tf.reduce_mean((TD)**2)] From 90e3fc3bd254f8751b14140341777af1f34dcc26 Mon Sep 17 00:00:00 2001 From: NoteDance <63648431+NoteDance@users.noreply.github.com> Date: Fri, 19 Jan 2024 22:14:38 +0800 Subject: [PATCH 252/337] Update PPO.py --- 7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py b/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py index c0319b11..9758690a 100644 --- a/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py +++ b/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py @@ -20,8 +20,8 @@ class critic: def __init__(self,state_dim,hidden_dim,action_dim): self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) self.bias1=tf.Variable(tf.random.normal([hidden_dim])) - self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) - self.bias2=tf.Variable(tf.random.normal([action_dim])) + self.weight2=tf.Variable(tf.random.normal([hidden_dim,1])) + self.bias2=tf.Variable(tf.random.normal([1])) self.param=[self.weight1,self.bias1,self.weight2,self.bias2] def fp(self,x): @@ -32,7 +32,7 @@ def fp(self,x): class PPO: def __init__(self,state_dim,hidden_dim,action_dim,clip_eps): self.nn=actor(state_dim,hidden_dim,action_dim) - self.critic=critic(state_dim,hidden_dim,action_dim) + self.critic=critic(state_dim,hidden_dim) self.actor_old=actor(state_dim,hidden_dim,action_dim) self.clip_eps=clip_eps self.param=[self.actor.param,self.critic.param] @@ -52,8 +52,8 @@ def env(self,a=None,initial=None): def loss(self,s,a,next_s,r,d): a=tf.expand_dims(a,axis=1) raito=tf.gather(self.actor.fp(s),a,axis=1,batch_dims=1)/tf.gather(self.actor_old.fp(s),a,axis=1,batch_dims=1) - value=tf.gather(self.critic.fp(s),a,axis=1,batch_dims=1) - value_tar=r+tf.gather(self.critic.fp(next_s),a,axis=1,batch_dims=1) + value=self.critic.fp(s) + value_tar=r+self.critic.fp(next_s) TD=r+0.98*value_tar*(1-d)-value clip=tf.clip_by_value(raito,clip_value_min=1-self.clip_eps,clip_value_max=1+self.clip_eps)*TD return [tf.reduce_mean(clip),tf.reduce_mean((TD)**2)] From 945bd8ff4c045fc7d7af8a2efeb2c412ae36ba7c Mon Sep 17 00:00:00 2001 From: NoteDance <63648431+NoteDance@users.noreply.github.com> Date: Fri, 19 Jan 2024 23:14:01 +0800 Subject: [PATCH 253/337] Update PPO.py --- 7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py b/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py index 71038b36..c1f4baef 100644 --- a/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py +++ b/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py @@ -69,3 +69,8 @@ def opt(self,gradient): self.opt.apply_gradients(zip(gradient[1],self.param[1])) self.actor_old.param=self.actor.param.copy() return + + + def update_param(self): + self.actor_old.param=self.actor.param.copy() + return From 6f242ddb9798ed1e901988ea7777c44f790ae0e9 Mon Sep 17 00:00:00 2001 From: NoteDance <63648431+NoteDance@users.noreply.github.com> Date: Fri, 19 Jan 2024 23:14:20 +0800 Subject: [PATCH 254/337] Update PPO.py --- 7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py b/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py index 9758690a..2f50899b 100644 --- a/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py +++ b/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py @@ -70,3 +70,8 @@ def opt(self,gradient): self.opt.opt(gradient[1],self.param[1]) self.actor_old.param=self.actor.param.copy() return + + + def update_param(self): + self.actor_old.param=self.actor.param.copy() + return From ebea79d2537532d65a511326181c0e4431641af8 Mon Sep 17 00:00:00 2001 From: NoteDance <63648431+NoteDance@users.noreply.github.com> Date: Fri, 19 Jan 2024 23:19:04 +0800 Subject: [PATCH 255/337] Update PPO.py --- 7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py b/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py index 2f50899b..526a8a5c 100644 --- a/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py +++ b/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py @@ -33,7 +33,7 @@ class PPO: def __init__(self,state_dim,hidden_dim,action_dim,clip_eps): self.nn=actor(state_dim,hidden_dim,action_dim) self.critic=critic(state_dim,hidden_dim) - self.actor_old=actor(state_dim,hidden_dim,action_dim) + self.actor=actor(state_dim,hidden_dim,action_dim) self.clip_eps=clip_eps self.param=[self.actor.param,self.critic.param] self.opt=SGD(param=self.param) @@ -51,7 +51,7 @@ def env(self,a=None,initial=None): def loss(self,s,a,next_s,r,d): a=tf.expand_dims(a,axis=1) - raito=tf.gather(self.actor.fp(s),a,axis=1,batch_dims=1)/tf.gather(self.actor_old.fp(s),a,axis=1,batch_dims=1) + raito=tf.gather(self.actor.fp(s),a,axis=1,batch_dims=1)/tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) value=self.critic.fp(s) value_tar=r+self.critic.fp(next_s) TD=r+0.98*value_tar*(1-d)-value @@ -73,5 +73,5 @@ def opt(self,gradient): def update_param(self): - self.actor_old.param=self.actor.param.copy() + self.nn.param=self.actor.param.copy() return From 2c2333990f837d749a241815772dad6960775d92 Mon Sep 17 00:00:00 2001 From: NoteDance <63648431+NoteDance@users.noreply.github.com> Date: Fri, 19 Jan 2024 23:19:18 +0800 Subject: [PATCH 256/337] Update PPO.py --- 7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py b/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py index c1f4baef..b2d84866 100644 --- a/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py +++ b/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py @@ -32,7 +32,7 @@ class PPO: def __init__(self,state_dim,hidden_dim,action_dim,clip_eps): self.nn=actor(state_dim,hidden_dim,action_dim) self.critic=critic(state_dim,hidden_dim) - self.actor_old=actor(state_dim,hidden_dim,action_dim) + self.actor=actor(state_dim,hidden_dim,action_dim) self.clip_eps=clip_eps self.param=[self.actor.param,self.critic.param] self.opt=tf.keras.optimizers.Adam() @@ -50,7 +50,7 @@ def env(self,a=None,initial=None): def loss(self,s,a,next_s,r,d): a=tf.expand_dims(a,axis=1) - raito=tf.gather(self.actor.fp(s),a,axis=1,batch_dims=1)/tf.gather(self.actor_old.fp(s),a,axis=1,batch_dims=1) + raito=tf.gather(self.actor.fp(s),a,axis=1,batch_dims=1)/tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) value=self.critic.fp(s) value_tar=r+self.critic.fp(next_s) TD=r+0.98*value_tar*(1-d)-value @@ -72,5 +72,5 @@ def opt(self,gradient): def update_param(self): - self.actor_old.param=self.actor.param.copy() + self.nn.param=self.actor.param.copy() return From a08ee985f322dac8d39ccf835e406415f43e9c64 Mon Sep 17 00:00:00 2001 From: NoteDance <63648431+NoteDance@users.noreply.github.com> Date: Sat, 20 Jan 2024 00:50:24 +0800 Subject: [PATCH 257/337] Update PPO.py --- 7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py b/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py index b2d84866..f3b35f6c 100644 --- a/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py +++ b/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py @@ -30,9 +30,10 @@ def fp(self,x): class PPO: def __init__(self,state_dim,hidden_dim,action_dim,clip_eps): + self.actor=actor(state_dim,hidden_dim,action_dim) self.nn=actor(state_dim,hidden_dim,action_dim) + self.nn.param=self.actor.param.copy() self.critic=critic(state_dim,hidden_dim) - self.actor=actor(state_dim,hidden_dim,action_dim) self.clip_eps=clip_eps self.param=[self.actor.param,self.critic.param] self.opt=tf.keras.optimizers.Adam() @@ -54,8 +55,10 @@ def loss(self,s,a,next_s,r,d): value=self.critic.fp(s) value_tar=r+self.critic.fp(next_s) TD=r+0.98*value_tar*(1-d)-value - clip=tf.clip_by_value(raito,clip_value_min=1-self.clip_eps,clip_value_max=1+self.clip_eps)*TD - return [tf.reduce_mean(clip),tf.reduce_mean((TD)**2)] + sur1=raito*TD + sur2=tf.clip_by_value(raito,clip_value_min=1-self.clip_eps,clip_value_max=1+self.clip_eps)*TD + clip_loss=-tf.math.minimum(sur1,sur2) + return [tf.reduce_mean(clip_loss),tf.reduce_mean((TD)**2)] def gradient(self,tape,loss): From ebea85da5db645d3fcb0d0518292cdaf4df9e8d4 Mon Sep 17 00:00:00 2001 From: NoteDance <63648431+NoteDance@users.noreply.github.com> Date: Sat, 20 Jan 2024 00:50:38 +0800 Subject: [PATCH 258/337] Update PPO.py --- 7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py b/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py index 526a8a5c..36988a22 100644 --- a/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py +++ b/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py @@ -31,9 +31,10 @@ def fp(self,x): class PPO: def __init__(self,state_dim,hidden_dim,action_dim,clip_eps): + self.actor=actor(state_dim,hidden_dim,action_dim) self.nn=actor(state_dim,hidden_dim,action_dim) + self.nn.param=self.actor.param.copy() self.critic=critic(state_dim,hidden_dim) - self.actor=actor(state_dim,hidden_dim,action_dim) self.clip_eps=clip_eps self.param=[self.actor.param,self.critic.param] self.opt=SGD(param=self.param) @@ -55,8 +56,10 @@ def loss(self,s,a,next_s,r,d): value=self.critic.fp(s) value_tar=r+self.critic.fp(next_s) TD=r+0.98*value_tar*(1-d)-value - clip=tf.clip_by_value(raito,clip_value_min=1-self.clip_eps,clip_value_max=1+self.clip_eps)*TD - return [tf.reduce_mean(clip),tf.reduce_mean((TD)**2)] + sur1=raito*TD + sur2=tf.clip_by_value(raito,clip_value_min=1-self.clip_eps,clip_value_max=1+self.clip_eps)*TD + clip_loss=-tf.math.minimum(sur1,sur2) + return [tf.reduce_mean(clip_loss),tf.reduce_mean((TD)**2)] def gradient(self,tape,loss): From 68bbd788e71f2b54eadb2befcdf1bdf92bcc97e4 Mon Sep 17 00:00:00 2001 From: NoteDance <63648431+NoteDance@users.noreply.github.com> Date: Sat, 20 Jan 2024 08:24:27 +0800 Subject: [PATCH 259/337] Update PPO.py --- 7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py b/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py index 36988a22..cd96d93c 100644 --- a/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py +++ b/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py @@ -54,8 +54,8 @@ def loss(self,s,a,next_s,r,d): a=tf.expand_dims(a,axis=1) raito=tf.gather(self.actor.fp(s),a,axis=1,batch_dims=1)/tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) value=self.critic.fp(s) - value_tar=r+self.critic.fp(next_s) - TD=r+0.98*value_tar*(1-d)-value + value_tar=r+0.98*self.critic.fp(next_s)*(1-d) + TD=value_tar-value sur1=raito*TD sur2=tf.clip_by_value(raito,clip_value_min=1-self.clip_eps,clip_value_max=1+self.clip_eps)*TD clip_loss=-tf.math.minimum(sur1,sur2) From 123474ef1cdfb3c9dad93e39d2dc50c0f7eeaaed Mon Sep 17 00:00:00 2001 From: NoteDance <63648431+NoteDance@users.noreply.github.com> Date: Sat, 20 Jan 2024 08:24:40 +0800 Subject: [PATCH 260/337] Update PPO.py --- 7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py b/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py index f3b35f6c..c16bd12c 100644 --- a/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py +++ b/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py @@ -53,8 +53,8 @@ def loss(self,s,a,next_s,r,d): a=tf.expand_dims(a,axis=1) raito=tf.gather(self.actor.fp(s),a,axis=1,batch_dims=1)/tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) value=self.critic.fp(s) - value_tar=r+self.critic.fp(next_s) - TD=r+0.98*value_tar*(1-d)-value + value_tar=r+0.98*self.critic.fp(next_s)*(1-d) + TD=value_tar-value sur1=raito*TD sur2=tf.clip_by_value(raito,clip_value_min=1-self.clip_eps,clip_value_max=1+self.clip_eps)*TD clip_loss=-tf.math.minimum(sur1,sur2) From e62012e15e5c8b18740907a99d55069bb2cdfa57 Mon Sep 17 00:00:00 2001 From: NoteDance <63648431+NoteDance@users.noreply.github.com> Date: Sat, 20 Jan 2024 13:41:56 +0800 Subject: [PATCH 261/337] Update and rename PPO.py to PPO_.py --- 7.0/neuralnetwork/RL/tensorflow/non_parallel/{PPO.py => PPO_.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename 7.0/neuralnetwork/RL/tensorflow/non_parallel/{PPO.py => PPO_.py} (100%) diff --git a/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py b/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO_.py similarity index 100% rename from 7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py rename to 7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO_.py From dd12fb82195f57bd6d66ba7869038b149d592cd4 Mon Sep 17 00:00:00 2001 From: NoteDance <63648431+NoteDance@users.noreply.github.com> Date: Sat, 20 Jan 2024 13:42:12 +0800 Subject: [PATCH 262/337] Add files via upload --- .../RL/tensorflow/non_parallel/PPO.py | 83 +++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py diff --git a/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py b/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py new file mode 100644 index 00000000..7e7f4204 --- /dev/null +++ b/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py @@ -0,0 +1,83 @@ +import tensorflow as tf +import gym + + +class actor: + def __init__(self,state_dim,hidden_dim,action_dim): + self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) + self.bias1=tf.Variable(tf.random.normal([hidden_dim])) + self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) + self.bias2=tf.Variable(tf.random.normal([action_dim])) + self.param=[self.weight1,self.bias1,self.weight2,self.bias2] + + def fp(self,x): + x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) + return tf.nn.softmax(tf.matmul(x,self.weight2)+self.bias2) + + +class critic: + def __init__(self,state_dim,hidden_dim): + self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) + self.bias1=tf.Variable(tf.random.normal([hidden_dim])) + self.weight2=tf.Variable(tf.random.normal([hidden_dim,1])) + self.bias2=tf.Variable(tf.random.normal([1])) + self.param=[self.weight1,self.bias1,self.weight2,self.bias2] + + def fp(self,x): + x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) + return tf.matmul(x,self.weight2)+self.bias2 + + +class PPO: + def __init__(self,state_dim,hidden_dim,action_dim,clip_eps,alpha): + self.actor=actor(state_dim,hidden_dim,action_dim) + self.nn=actor(state_dim,hidden_dim,action_dim) + self.nn.param=self.actor.param.copy() + self.critic=critic(state_dim,hidden_dim) + self.clip_eps=clip_eps + self.alpha=alpha + self.param=[self.actor.param,self.critic.param] + self.opt=tf.keras.optimizers.Adam() + self.genv=gym.make('CartPole-v0') + + + def env(self,a=None,initial=None): + if initial==True: + state=self.genv.reset(seed=0) + return state + else: + next_state,reward,done,_=self.genv.step(a) + return next_state,reward,done + + + def loss(self,s,a,next_s,r,d): + a=tf.expand_dims(a,axis=1) + raito=tf.gather(self.actor.fp(s),a,axis=1,batch_dims=1)/tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) + value=self.critic.fp(s) + value_tar=r+0.98*self.critic.fp(next_s)*(1-d) + TD=value_tar-value + sur1=raito*TD + sur2=tf.clip_by_value(raito,clip_value_min=1-self.clip_eps,clip_value_max=1+self.clip_eps)*TD + clip_loss=-tf.math.minimum(sur1,sur2) + entropy=self.actor.fp(s)*tf.math.log(self.actor.fp(s)+1e-8) + clip_loss=clip_loss-self.alpha*entropy + return [tf.reduce_mean(clip_loss),tf.reduce_mean((TD)**2)] + + + + def gradient(self,tape,loss): + actor_gradient=tape.gradient(loss[0],self.param[0]) + critic_gradient=tape.gradient(loss[1],self.param[1]) + return [actor_gradient,critic_gradient] + + + def opt(self,gradient): + self.opt.apply_gradients(zip(gradient[0],self.param[0])) + self.opt.apply_gradients(zip(gradient[1],self.param[1])) + self.actor_old.param=self.actor.param.copy() + return + + + def update_param(self): + self.nn.param=self.actor.param.copy() + return \ No newline at end of file From 8f94ffc2a5fc5cc7ef09a3de452c0e4220903f76 Mon Sep 17 00:00:00 2001 From: NoteDance <63648431+NoteDance@users.noreply.github.com> Date: Sat, 20 Jan 2024 13:42:29 +0800 Subject: [PATCH 263/337] Update and rename PPO.py to PPO_.py --- 7.0/neuralnetwork/RL/tensorflow/parallel/{PPO.py => PPO_.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename 7.0/neuralnetwork/RL/tensorflow/parallel/{PPO.py => PPO_.py} (100%) diff --git a/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py b/7.0/neuralnetwork/RL/tensorflow/parallel/PPO_.py similarity index 100% rename from 7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py rename to 7.0/neuralnetwork/RL/tensorflow/parallel/PPO_.py From 7f4e056dc983d3562b9d2e052d8320de796c5fd5 Mon Sep 17 00:00:00 2001 From: NoteDance <63648431+NoteDance@users.noreply.github.com> Date: Sat, 20 Jan 2024 13:42:45 +0800 Subject: [PATCH 264/337] Add files via upload --- .../RL/tensorflow/parallel/PPO.py | 82 +++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py diff --git a/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py b/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py new file mode 100644 index 00000000..0cb26984 --- /dev/null +++ b/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py @@ -0,0 +1,82 @@ +import tensorflow as tf +import gym +from Note.nn.parallel.optimizer import SGD + + +class actor: + def __init__(self,state_dim,hidden_dim,action_dim): + self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) + self.bias1=tf.Variable(tf.random.normal([hidden_dim])) + self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) + self.bias2=tf.Variable(tf.random.normal([action_dim])) + self.param=[self.weight1,self.bias1,self.weight2,self.bias2] + + def fp(self,x): + x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) + return tf.nn.softmax(tf.matmul(x,self.weight2)+self.bias2) + + +class critic: + def __init__(self,state_dim,hidden_dim,action_dim): + self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) + self.bias1=tf.Variable(tf.random.normal([hidden_dim])) + self.weight2=tf.Variable(tf.random.normal([hidden_dim,1])) + self.bias2=tf.Variable(tf.random.normal([1])) + self.param=[self.weight1,self.bias1,self.weight2,self.bias2] + + def fp(self,x): + x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) + return tf.matmul(x,self.weight2)+self.bias2 + + +class PPO: + def __init__(self,state_dim,hidden_dim,action_dim,clip_eps,alpha): + self.actor=actor(state_dim,hidden_dim,action_dim) + self.nn=actor(state_dim,hidden_dim,action_dim) + self.nn.param=self.actor.param.copy() + self.critic=critic(state_dim,hidden_dim) + self.clip_eps=clip_eps + self.param=[self.actor.param,self.critic.param] + self.opt=SGD(param=self.param) + self.genv=[gym.make('CartPole-v0') for _ in range(5)] + + + def env(self,a=None,initial=None): + if initial==True: + state=self.genv.reset(seed=0) + return state + else: + next_state,reward,done,_=self.genv.step(a) + return next_state,reward,done + + + def loss(self,s,a,next_s,r,d): + a=tf.expand_dims(a,axis=1) + raito=tf.gather(self.actor.fp(s),a,axis=1,batch_dims=1)/tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) + value=self.critic.fp(s) + value_tar=r+0.98*self.critic.fp(next_s)*(1-d) + TD=value_tar-value + sur1=raito*TD + sur2=tf.clip_by_value(raito,clip_value_min=1-self.clip_eps,clip_value_max=1+self.clip_eps)*TD + clip_loss=-tf.math.minimum(sur1,sur2) + entropy=self.actor.fp(s)*tf.math.log(self.actor.fp(s)+1e-8) + clip_loss=clip_loss-self.alpha*entropy + return [tf.reduce_mean(clip_loss),tf.reduce_mean((TD)**2)] + + + def gradient(self,tape,loss): + actor_gradient=tape.gradient(loss[0],self.param[0]) + critic_gradient=tape.gradient(loss[1],self.param[1]) + return [actor_gradient,critic_gradient] + + + def opt(self,gradient): + self.opt.opt(gradient[0],self.param[0]) + self.opt.opt(gradient[1],self.param[1]) + self.actor_old.param=self.actor.param.copy() + return + + + def update_param(self): + self.nn.param=self.actor.param.copy() + return \ No newline at end of file From 46614abea42a74435d864d50c2b9adc473be75c5 Mon Sep 17 00:00:00 2001 From: NoteDance <63648431+NoteDance@users.noreply.github.com> Date: Sat, 20 Jan 2024 14:01:29 +0800 Subject: [PATCH 265/337] Update PPO.py --- 7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py b/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py index 0cb26984..2f1f2a70 100644 --- a/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py +++ b/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py @@ -52,14 +52,16 @@ def env(self,a=None,initial=None): def loss(self,s,a,next_s,r,d): a=tf.expand_dims(a,axis=1) - raito=tf.gather(self.actor.fp(s),a,axis=1,batch_dims=1)/tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) + action_prob=tf.gather(self.actor.fp(s),a,axis=1,batch_dims=1) + action_prob_old=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) + raito=action_prob/action_prob_old value=self.critic.fp(s) value_tar=r+0.98*self.critic.fp(next_s)*(1-d) TD=value_tar-value sur1=raito*TD sur2=tf.clip_by_value(raito,clip_value_min=1-self.clip_eps,clip_value_max=1+self.clip_eps)*TD clip_loss=-tf.math.minimum(sur1,sur2) - entropy=self.actor.fp(s)*tf.math.log(self.actor.fp(s)+1e-8) + entropy=action_prob*tf.math.log(action_prob+1e-8) clip_loss=clip_loss-self.alpha*entropy return [tf.reduce_mean(clip_loss),tf.reduce_mean((TD)**2)] @@ -79,4 +81,4 @@ def opt(self,gradient): def update_param(self): self.nn.param=self.actor.param.copy() - return \ No newline at end of file + return From 12bd6db67a5d4f8b060392774bcc1b8297c8d9aa Mon Sep 17 00:00:00 2001 From: NoteDance <63648431+NoteDance@users.noreply.github.com> Date: Sat, 20 Jan 2024 14:01:43 +0800 Subject: [PATCH 266/337] Update PPO.py --- 7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py b/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py index 7e7f4204..6321c346 100644 --- a/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py +++ b/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py @@ -52,14 +52,16 @@ def env(self,a=None,initial=None): def loss(self,s,a,next_s,r,d): a=tf.expand_dims(a,axis=1) - raito=tf.gather(self.actor.fp(s),a,axis=1,batch_dims=1)/tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) + action_prob=tf.gather(self.actor.fp(s),a,axis=1,batch_dims=1) + action_prob_old=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) + raito=action_prob/action_prob_old value=self.critic.fp(s) value_tar=r+0.98*self.critic.fp(next_s)*(1-d) TD=value_tar-value sur1=raito*TD sur2=tf.clip_by_value(raito,clip_value_min=1-self.clip_eps,clip_value_max=1+self.clip_eps)*TD clip_loss=-tf.math.minimum(sur1,sur2) - entropy=self.actor.fp(s)*tf.math.log(self.actor.fp(s)+1e-8) + entropy=action_prob*tf.math.log(action_prob+1e-8) clip_loss=clip_loss-self.alpha*entropy return [tf.reduce_mean(clip_loss),tf.reduce_mean((TD)**2)] @@ -80,4 +82,4 @@ def opt(self,gradient): def update_param(self): self.nn.param=self.actor.param.copy() - return \ No newline at end of file + return From 423d22a494425b36f2e1b66be7f1feff0a08c5c5 Mon Sep 17 00:00:00 2001 From: NoteDance <63648431+NoteDance@users.noreply.github.com> Date: Sun, 21 Jan 2024 09:23:04 +0800 Subject: [PATCH 267/337] Update PPO_.py --- 7.0/neuralnetwork/RL/tensorflow/parallel/PPO_.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/7.0/neuralnetwork/RL/tensorflow/parallel/PPO_.py b/7.0/neuralnetwork/RL/tensorflow/parallel/PPO_.py index cd96d93c..31498408 100644 --- a/7.0/neuralnetwork/RL/tensorflow/parallel/PPO_.py +++ b/7.0/neuralnetwork/RL/tensorflow/parallel/PPO_.py @@ -41,12 +41,12 @@ def __init__(self,state_dim,hidden_dim,action_dim,clip_eps): self.genv=[gym.make('CartPole-v0') for _ in range(5)] - def env(self,a=None,initial=None): + def env(self,a=None,p=None,initial=None): if initial==True: - state=self.genv.reset(seed=0) + state=self.genv[p].reset(seed=0) return state else: - next_state,reward,done,_=self.genv.step(a) + next_state,reward,done,_=self.genv[p].step(a) return next_state,reward,done From 86726b90f6573b896394d858eb924282a18b2a3a Mon Sep 17 00:00:00 2001 From: NoteDance <63648431+NoteDance@users.noreply.github.com> Date: Sun, 21 Jan 2024 09:23:17 +0800 Subject: [PATCH 268/337] Update PPO.py --- 7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py b/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py index 2f1f2a70..66a9441f 100644 --- a/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py +++ b/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py @@ -41,12 +41,12 @@ def __init__(self,state_dim,hidden_dim,action_dim,clip_eps,alpha): self.genv=[gym.make('CartPole-v0') for _ in range(5)] - def env(self,a=None,initial=None): + def env(self,a=None,p=None,initial=None): if initial==True: - state=self.genv.reset(seed=0) + state=self.genv[p].reset(seed=0) return state else: - next_state,reward,done,_=self.genv.step(a) + next_state,reward,done,_=self.genv[p].step(a) return next_state,reward,done From af241d02ed19b87ea0a4071d1b7334b35c84d263 Mon Sep 17 00:00:00 2001 From: NoteDance <63648431+NoteDance@users.noreply.github.com> Date: Sun, 21 Jan 2024 09:23:53 +0800 Subject: [PATCH 269/337] Update and rename PPO_.py to PPO_eps.py --- 7.0/neuralnetwork/RL/tensorflow/parallel/{PPO_.py => PPO_eps.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename 7.0/neuralnetwork/RL/tensorflow/parallel/{PPO_.py => PPO_eps.py} (100%) diff --git a/7.0/neuralnetwork/RL/tensorflow/parallel/PPO_.py b/7.0/neuralnetwork/RL/tensorflow/parallel/PPO_eps.py similarity index 100% rename from 7.0/neuralnetwork/RL/tensorflow/parallel/PPO_.py rename to 7.0/neuralnetwork/RL/tensorflow/parallel/PPO_eps.py From 0ff6de5c278d2792f2b5c778c5f6ee312221bdad Mon Sep 17 00:00:00 2001 From: NoteDance <63648431+NoteDance@users.noreply.github.com> Date: Sun, 21 Jan 2024 09:24:09 +0800 Subject: [PATCH 270/337] Update and rename PPO_.py to PPO_eps.py --- .../RL/tensorflow/non_parallel/{PPO_.py => PPO_eps.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename 7.0/neuralnetwork/RL/tensorflow/non_parallel/{PPO_.py => PPO_eps.py} (100%) diff --git a/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO_.py b/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO_eps.py similarity index 100% rename from 7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO_.py rename to 7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO_eps.py From 641c0e8655a75b294f6712c16874ac31624572fe Mon Sep 17 00:00:00 2001 From: NoteDance <63648431+NoteDance@users.noreply.github.com> Date: Thu, 8 Feb 2024 19:47:47 +0800 Subject: [PATCH 271/337] Add files via upload --- 7.0/neuralnetwork/DL/tensorflow/parallel/cnn.py | 14 +++++++------- 7.0/neuralnetwork/DL/tensorflow/parallel/nn.py | 10 +++++----- 7.0/neuralnetwork/DL/tensorflow/parallel/nn_acc.py | 10 +++++----- .../DL/tensorflow/parallel/nn_attenuate.py | 11 +++++------ .../DL/tensorflow/parallel/nn_clipping.py | 10 +++++----- .../DL/tensorflow/parallel/nn_device.py | 10 +++++----- 6 files changed, 32 insertions(+), 33 deletions(-) diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/cnn.py b/7.0/neuralnetwork/DL/tensorflow/parallel/cnn.py index 8e130ccd..f8153fc4 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/cnn.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/cnn.py @@ -29,13 +29,13 @@ def build(self): def fp(self,data): # Perform forward propagation on the input data - x=self.conv1.output(data) # First convolutional layer + x=self.conv1(data) # First convolutional layer x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # First max pooling layer - x=self.conv2.output(x) # Second convolutional layer + x=self.conv2(x) # Second convolutional layer x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # Second max pooling layer - x=self.conv3.output(x) # Third convolutional layer - x=self.flatten.output(x) # Flatten the output to a vector - x=self.dense1.output(x) # First dense layer with relu activation + x=self.conv3(x) # Third convolutional layer + x=self.flatten(x) # Flatten the output to a vector + x=self.dense1(x) # First dense layer with relu activation output=self.dense2.output(x) # Output layer with linear activation return output @@ -47,5 +47,5 @@ def loss(self,output,labels): def opt(self,gradient): # Perform optimization on the parameters using the gradient and the batch count - param=self.optimizer.opt(gradient,self.param,self.bc[0]) - return param + param=self.optimizer(gradient,self.param,self.bc[0]) + return param \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn.py index 87f71817..709c833b 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/nn.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/nn.py @@ -24,9 +24,9 @@ def build(self): def fp(self,data): # Perform forward propagation on the input data - data=self.flatten.output(data) - output1=self.layer1.output(data) - output2=self.layer2.output(output1) + data=self.flatten(data) + output1=self.layer1(data) + output2=self.layer2(output1) return output2 @@ -37,5 +37,5 @@ def loss(self,output,labels): def opt(self,gradient): # Perform optimization on the parameters using the gradient - param=self.optimizer.opt(gradient,self.param) - return param + param=self.optimizer(gradient,self.param) + return param \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_acc.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_acc.py index 51895f99..baf05517 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_acc.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_acc.py @@ -25,9 +25,9 @@ def build(self): def fp(self,data): # Perform forward propagation on the input data - data=self.flatten.output(data) - output1=self.layer1.output(data) - output2=self.layer2.output(output1) + data=self.flatten(data) + output1=self.layer1(data) + output2=self.layer2(output1) return output2 @@ -42,5 +42,5 @@ def accuracy(self,output,labels): #accuracy function,kernel uses it to calculate def opt(self,gradient): # Perform optimization on the parameters using the gradient - param=self.optimizer.opt(gradient,self.param) - return param + param=self.optimizer(gradient,self.param) + return param \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_attenuate.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_attenuate.py index 9f71ddd0..892208a2 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_attenuate.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_attenuate.py @@ -26,9 +26,9 @@ def build(self): def fp(self,data): # Perform forward propagation on the input data - data=self.flatten.output(data) # Flatten the data to a one-dimensional vector - output1=self.layer1.output(data) # Apply the first layer to the data and get the output - output2=self.layer2.output(output1) # Apply the second layer to the output of the first layer and get the final output + data=self.flatten(data) # Flatten the data to a one-dimensional vector + output1=self.layer1(data) # Apply the first layer to the data and get the output + output2=self.layer2(output1) # Apply the second layer to the output of the first layer and get the final output return output2 @@ -47,11 +47,10 @@ def attenuate(self,gradient,p): #gradient attenuation function,kernel uses it t gradient_flat=nest.flatten(gradient) # Flatten the gradient to a one-dimensional vector for i in range(len(gradient_flat)): #self.opt_counter:optimization counter gradient_flat[i]=tf.cast(ac,gradient_flat[i].dtype)*gradient_flat[i] #p:process number - gradient=nest.pack_sequence_as(gradient,gradient_flat) # Restore the gradient to its original shape return gradient def opt(self,gradient): # Perform optimization on the parameters using the gradient - param=self.optimizer.opt(gradient,self.param) # Use the optimizer to update the parameters - return param + param=self.optimizer(gradient,self.param) # Use the optimizer to update the parameters + return param \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_clipping.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_clipping.py index 46aa5eb9..7a523f86 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_clipping.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_clipping.py @@ -25,9 +25,9 @@ def build(self): def fp(self,data): # Perform forward propagation on the input data - data=self.flatten.output(data) - output1=self.layer1.output(data) - output2=self.layer2.output(output1) + data=self.flatten(data) + output1=self.layer1(data) + output2=self.layer2(output1) return output2 @@ -45,5 +45,5 @@ def gradient(self,tape,loss): def opt(self,gradient): # Perform optimization on the parameters using the gradient - param=self.optimizer.opt(gradient,self.param) - return param + param=self.optimizer(gradient,self.param) + return param \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_device.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_device.py index 1f24083b..a8bb0970 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_device.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_device.py @@ -24,9 +24,9 @@ def build(self): # build function, kernel uses it to create the network layers def fp(self,data,p): # forward propagation function, kernel uses it for forward propagation with tf.device(assign_device(p,'GPU')): # assign the device according to the process index p - data=self.flatten.output(data) # flatten the data to a vector of 784 elements - output1=self.layer1.output(data) # pass the data through the first layer and get the output - output2=self.layer2.output(output1) # pass the output of the first layer through the second layer and get the final output logits + data=self.flatten(data) # flatten the data to a vector of 784 elements + output1=self.layer1(data) # pass the data through the first layer and get the output + output2=self.layer2(output1) # pass the output of the first layer through the second layer and get the final output logits return output2 @@ -46,5 +46,5 @@ def GradientTape(self,data,labels,p): def opt(self,gradient,p): # optimization function, kernel uses it to optimize parameter # Perform optimization on the parameters using the gradient with tf.device(assign_device(p,'GPU')): # assign the device according to the process index p - param=self.optimizer.opt(gradient,self.param) # apply the Note's momentum optimizer to update the parameters using the gradient - return param + param=self.optimizer(gradient,self.param) # apply the Note's momentum optimizer to update the parameters using the gradient + return param \ No newline at end of file From 0b9fa3c217d1dbd7fa9dcad0ae69f3c44826546d Mon Sep 17 00:00:00 2001 From: NoteDance <63648431+NoteDance@users.noreply.github.com> Date: Thu, 8 Feb 2024 19:50:20 +0800 Subject: [PATCH 272/337] Add files via upload --- .../DL/tensorflow/non_parallel/layer/cnn.py | 10 +++++----- .../DL/tensorflow/non_parallel/layer/nn.py | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/7.0/neuralnetwork/DL/tensorflow/non_parallel/layer/cnn.py b/7.0/neuralnetwork/DL/tensorflow/non_parallel/layer/cnn.py index 0ee576cb..00f1ad0b 100644 --- a/7.0/neuralnetwork/DL/tensorflow/non_parallel/layer/cnn.py +++ b/7.0/neuralnetwork/DL/tensorflow/non_parallel/layer/cnn.py @@ -32,13 +32,13 @@ def build(self): def fp(self,data): # Perform forward propagation on the input data - x=self.conv1.output(data) # First convolutional layer + x=self.conv1(data) # First convolutional layer x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # First max pooling layer - x=self.conv2.output(x) # Second convolutional layer + x=self.conv2(x) # Second convolutional layer x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # Second max pooling layer - x=self.conv3.output(x) # Third convolutional layer - x=self.flatten.output(x) # Flatten the output to a vector - x=self.dense1.output(x) # First dense layer with relu activation + x=self.conv3(x) # Third convolutional layer + x=self.flatten(x) # Flatten the output to a vector + x=self.dense1(x) # First dense layer with relu activation output=self.dense2.output(x) # Output layer with linear activation return output diff --git a/7.0/neuralnetwork/DL/tensorflow/non_parallel/layer/nn.py b/7.0/neuralnetwork/DL/tensorflow/non_parallel/layer/nn.py index ae9ff128..65cde43a 100644 --- a/7.0/neuralnetwork/DL/tensorflow/non_parallel/layer/nn.py +++ b/7.0/neuralnetwork/DL/tensorflow/non_parallel/layer/nn.py @@ -27,9 +27,9 @@ def build(self): def fp(self,data): # Perform forward propagation on the input data - data=self.flatten.output(data) - output1=self.layer1.output(data) - output2=self.layer2.output(output1) + data=self.flatten(data) + output1=self.layer1(data) + output2=self.layer2(output1) return output2 From 5b8b959f857b8e1d9fe059fa4701c780acda5ce7 Mon Sep 17 00:00:00 2001 From: NoteDance <63648431+NoteDance@users.noreply.github.com> Date: Tue, 13 Feb 2024 22:16:27 +0800 Subject: [PATCH 273/337] Update nn_device.py --- 7.0/neuralnetwork/DL/tensorflow/parallel/nn_device.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_device.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_device.py index a8bb0970..e072f845 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_device.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_device.py @@ -40,11 +40,11 @@ def GradientTape(self,data,labels,p): with tf.GradientTape(persistent=True) as tape: output=self.fp(data,p) loss=self.loss(output,labels,p) - return tape,output,loss + return tape,output,loss def opt(self,gradient,p): # optimization function, kernel uses it to optimize parameter # Perform optimization on the parameters using the gradient with tf.device(assign_device(p,'GPU')): # assign the device according to the process index p param=self.optimizer(gradient,self.param) # apply the Note's momentum optimizer to update the parameters using the gradient - return param \ No newline at end of file + return param From cbcee48f023ec26ffb89a9e040d98bfbedc93e45 Mon Sep 17 00:00:00 2001 From: NoteDance Date: Mon, 1 Jul 2024 14:03:10 +0800 Subject: [PATCH 274/337] Add files via upload --- 7.0/neuralnetwork/DL/tensorflow/non_parallel/layer/cnn.py | 4 ++-- 7.0/neuralnetwork/DL/tensorflow/non_parallel/layer/nn.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/7.0/neuralnetwork/DL/tensorflow/non_parallel/layer/cnn.py b/7.0/neuralnetwork/DL/tensorflow/non_parallel/layer/cnn.py index 00f1ad0b..cf396557 100644 --- a/7.0/neuralnetwork/DL/tensorflow/non_parallel/layer/cnn.py +++ b/7.0/neuralnetwork/DL/tensorflow/non_parallel/layer/cnn.py @@ -2,7 +2,7 @@ from Note.nn.layer.conv2d import conv2d from Note.nn.layer.dense import dense from Note.nn.layer.flatten import flatten -from Note.nn.Module import Module +from Note.nn.Model import Model """ This is an example of using the Note layer module. @@ -26,7 +26,7 @@ def build(self): self.dense1=dense(64,activation='relu') self.dense2=dense(10) # Store the parameters of the layers in a list - self.param=Module.param + self.param=Model.param return diff --git a/7.0/neuralnetwork/DL/tensorflow/non_parallel/layer/nn.py b/7.0/neuralnetwork/DL/tensorflow/non_parallel/layer/nn.py index 65cde43a..7dc63176 100644 --- a/7.0/neuralnetwork/DL/tensorflow/non_parallel/layer/nn.py +++ b/7.0/neuralnetwork/DL/tensorflow/non_parallel/layer/nn.py @@ -1,7 +1,7 @@ import tensorflow as tf from Note.nn.layer.dense import dense from Note.nn.layer.flatten import flatten -from Note.nn.Module import Module +from Note.nn.Model import Model """ This is an example of using the Note layer module. @@ -21,7 +21,7 @@ def build(self): self.layer2=dense(10) self.flatten=flatten() # Store the parameters of the layers in a list - self.param=Module.param + self.param=Model.param return From 9f600c9ebacb4c0cb1c269b63c2d2ce673d9e0bd Mon Sep 17 00:00:00 2001 From: NoteDance Date: Mon, 1 Jul 2024 14:03:36 +0800 Subject: [PATCH 275/337] Add files via upload --- 7.0/neuralnetwork/DL/tensorflow/parallel/cnn.py | 4 ++-- 7.0/neuralnetwork/DL/tensorflow/parallel/nn.py | 4 ++-- 7.0/neuralnetwork/DL/tensorflow/parallel/nn_acc.py | 4 ++-- 7.0/neuralnetwork/DL/tensorflow/parallel/nn_attenuate.py | 4 ++-- 7.0/neuralnetwork/DL/tensorflow/parallel/nn_clipping.py | 4 ++-- 7.0/neuralnetwork/DL/tensorflow/parallel/nn_device.py | 6 +++--- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/cnn.py b/7.0/neuralnetwork/DL/tensorflow/parallel/cnn.py index f8153fc4..6e084ee8 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/cnn.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/cnn.py @@ -3,7 +3,7 @@ from Note.nn.layer.dense import dense from Note.nn.layer.flatten import flatten from Note.nn.parallel.optimizer import Adam -from Note.nn.Module import Module +from Note.nn.Model import Model # Define a convolutional neural network class class cnn: @@ -23,7 +23,7 @@ def build(self): self.dense2=dense(10,64) self.optimizer=Adam() # Store the parameters of the layers in a list - self.param=Module.param + self.param=Model.param return diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn.py index 709c833b..db171189 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/nn.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/nn.py @@ -2,7 +2,7 @@ from Note.nn.layer.dense import dense from Note.nn.parallel.optimizer import SGD from Note.nn.layer.flatten import flatten -from Note.nn.Module import Module +from Note.nn.Model import Model # Define a neural network class class nn: @@ -18,7 +18,7 @@ def build(self): self.flatten=flatten() self.optimizer=SGD() # Store the parameters of the layers in a list - self.param=Module.param + self.param=Model.param return diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_acc.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_acc.py index baf05517..cd3c8026 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_acc.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_acc.py @@ -3,7 +3,7 @@ from Note.nn.parallel.optimizer import SGD from Note.nn.layer.flatten import flatten from Note.nn.accuracy import sparse_categorical_accuracy -from Note.nn.Module import Module +from Note.nn.Model import Model # Define a neural network class class nn: @@ -19,7 +19,7 @@ def build(self): self.flatten=flatten() self.optimizer=SGD() # Store the parameters of the layers in a list - self.param=Module.param + self.param=Model.param return diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_attenuate.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_attenuate.py index 892208a2..41f77f62 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_attenuate.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_attenuate.py @@ -4,7 +4,7 @@ from Note.nn.parallel.optimizer import SGD from Note.nn.layer.flatten import flatten from Note.nn.accuracy import sparse_categorical_accuracy -from Note.nn.Module import Module +from Note.nn.Model import Model # Define a neural network class with gradient attenuation class nn: @@ -20,7 +20,7 @@ def build(self): self.flatten=flatten() self.optimizer=SGD() # Store the parameters of the layers in a list - self.param=Module.param + self.param=Model.param return diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_clipping.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_clipping.py index 7a523f86..4929a49b 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_clipping.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_clipping.py @@ -3,7 +3,7 @@ from Note.nn.parallel.optimizer import SGD from Note.nn.layer.flatten import flatten from Note.nn.gradient_clipping import gradient_clipping -from Note.nn.Module import Module +from Note.nn.Model import Model # Define a neural network class class nn: @@ -19,7 +19,7 @@ def build(self): self.flatten=flatten() self.optimizer=SGD() # Store the parameters of the layers in a list - self.param=Module.param + self.param=Model.param return diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_device.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_device.py index e072f845..7935af2e 100644 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_device.py +++ b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_device.py @@ -3,7 +3,7 @@ from Note.nn.layer.flatten import flatten # import Note's flatten layer function from Note.nn.parallel.optimizer import SGD # import Note's momentum optimizer module from Note.nn.parallel.assign_device import assign_device # import the function to assign device according to the process index and the device type -from Note.nn.Module import Module +from Note.nn.Model import Model class nn: # A neural network class example, allocate device for multiple threads @@ -18,7 +18,7 @@ def build(self): # build function, kernel uses it to create the network layers self.flatten=flatten() self.optimizer=SGD() # Store the parameters of the layers in a list - self.param=Module.param # parameter list of both layers, kernel uses it list for backpropagation + self.param=Model.param # parameter list of both layers, kernel uses it list for backpropagation return @@ -47,4 +47,4 @@ def opt(self,gradient,p): # optimization function, kernel uses it to optimize pa # Perform optimization on the parameters using the gradient with tf.device(assign_device(p,'GPU')): # assign the device according to the process index p param=self.optimizer(gradient,self.param) # apply the Note's momentum optimizer to update the parameters using the gradient - return param + return param \ No newline at end of file From 69d1e4163e3238f53154a8c6f3f4972dd1d112df Mon Sep 17 00:00:00 2001 From: NoteDance Date: Tue, 16 Jul 2024 16:31:43 +0800 Subject: [PATCH 276/337] Update PPO_eps.py --- 7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO_eps.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO_eps.py b/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO_eps.py index c16bd12c..304419bf 100644 --- a/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO_eps.py +++ b/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO_eps.py @@ -1,4 +1,5 @@ import tensorflow as tf +from Note import nn import gym @@ -75,5 +76,5 @@ def opt(self,gradient): def update_param(self): - self.nn.param=self.actor.param.copy() + nn.assign_param(self.nn.param, self.actor.param.copy()) return From 43bf5cd7c21a5fabe8ce27d7ec09d8eabfaa9a48 Mon Sep 17 00:00:00 2001 From: NoteDance Date: Tue, 16 Jul 2024 16:31:54 +0800 Subject: [PATCH 277/337] Update PPO.py --- 7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py b/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py index 6321c346..150b4f5c 100644 --- a/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py +++ b/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py @@ -1,4 +1,5 @@ import tensorflow as tf +from Note import nn import gym @@ -81,5 +82,5 @@ def opt(self,gradient): def update_param(self): - self.nn.param=self.actor.param.copy() + nn.assign_param(self.nn.param, self.actor.param.copy()) return From 6b0b0f63a71862eef5f61fa3bdceddf847c1b8f2 Mon Sep 17 00:00:00 2001 From: NoteDance Date: Tue, 16 Jul 2024 16:32:04 +0800 Subject: [PATCH 278/337] Update DQN_pr.py --- 7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN_pr.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN_pr.py b/7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN_pr.py index 77e8b8a9..7436ee7f 100644 --- a/7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN_pr.py +++ b/7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN_pr.py @@ -1,4 +1,5 @@ import tensorflow as tf +from Note import nn import gym import Note.create.RL.rl.prioritized_replay as pr @@ -55,5 +56,5 @@ def loss(self,s,a,next_s,r,d): def update_param(self): - self.target_q_net.param=self.param.copy() - return \ No newline at end of file + nn.assign_param(self.target_q_net.param, self.param.copy()) + return From ed9551d7139d86f24603f4271fda4072c5d893bf Mon Sep 17 00:00:00 2001 From: NoteDance Date: Tue, 16 Jul 2024 16:32:16 +0800 Subject: [PATCH 279/337] Update DDPG.py --- 7.0/neuralnetwork/RL/tensorflow/non_parallel/DDPG.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/7.0/neuralnetwork/RL/tensorflow/non_parallel/DDPG.py b/7.0/neuralnetwork/RL/tensorflow/non_parallel/DDPG.py index d3666ee1..5779cf40 100644 --- a/7.0/neuralnetwork/RL/tensorflow/non_parallel/DDPG.py +++ b/7.0/neuralnetwork/RL/tensorflow/non_parallel/DDPG.py @@ -78,7 +78,7 @@ def loss(self,s,a,next_s,r,d): # loss function, kernel uses it to calculate los def update_param(self): # update function, kernel uses it to update parameter for target_param,param in zip(self.target_actor.param,self.actor.param): # for each pair of parameters in target actor network and actor network - target_param=target_param*(1.0-self.tau)+param*self.tau # update the target parameter using soft update with factor tau + target_param.assign(target_param*(1.0-self.tau)+param*self.tau) # update the target parameter using soft update with factor tau for target_param,param in zip(self.target_critic.param,self.critic.param): # for each pair of parameters in target critic network and critic network - target_param=target_param*(1.0-self.tau)+param*self.tau # update the target parameter using soft update with factor tau - return \ No newline at end of file + target_param.assign(target_param*(1.0-self.tau)+param*self.tau) # update the target parameter using soft update with factor tau + return From 675adbaa21f623f16f214e256fb408c17da11014 Mon Sep 17 00:00:00 2001 From: NoteDance Date: Tue, 16 Jul 2024 16:33:43 +0800 Subject: [PATCH 280/337] Update PPO_eps.py --- 7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO_eps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO_eps.py b/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO_eps.py index 304419bf..fb00b9cd 100644 --- a/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO_eps.py +++ b/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO_eps.py @@ -33,7 +33,7 @@ class PPO: def __init__(self,state_dim,hidden_dim,action_dim,clip_eps): self.actor=actor(state_dim,hidden_dim,action_dim) self.nn=actor(state_dim,hidden_dim,action_dim) - self.nn.param=self.actor.param.copy() + nn.assign_param(self.nn.param,self.actor.param.copy()) self.critic=critic(state_dim,hidden_dim) self.clip_eps=clip_eps self.param=[self.actor.param,self.critic.param] From 35b35d6f74d715c2aa41c9ecc31cefe421dc3d5e Mon Sep 17 00:00:00 2001 From: NoteDance Date: Tue, 16 Jul 2024 16:33:54 +0800 Subject: [PATCH 281/337] Update PPO.py --- 7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py b/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py index 150b4f5c..f65a524b 100644 --- a/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py +++ b/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py @@ -33,7 +33,7 @@ class PPO: def __init__(self,state_dim,hidden_dim,action_dim,clip_eps,alpha): self.actor=actor(state_dim,hidden_dim,action_dim) self.nn=actor(state_dim,hidden_dim,action_dim) - self.nn.param=self.actor.param.copy() + nn.assign_param(self.nn.param,self.actor.param.copy()) self.critic=critic(state_dim,hidden_dim) self.clip_eps=clip_eps self.alpha=alpha From 40a0606694174e8cf4d0a708c0010b8693234b46 Mon Sep 17 00:00:00 2001 From: NoteDance Date: Tue, 16 Jul 2024 16:34:06 +0800 Subject: [PATCH 282/337] Update DDPG.py --- 7.0/neuralnetwork/RL/tensorflow/non_parallel/DDPG.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/7.0/neuralnetwork/RL/tensorflow/non_parallel/DDPG.py b/7.0/neuralnetwork/RL/tensorflow/non_parallel/DDPG.py index 5779cf40..b90af114 100644 --- a/7.0/neuralnetwork/RL/tensorflow/non_parallel/DDPG.py +++ b/7.0/neuralnetwork/RL/tensorflow/non_parallel/DDPG.py @@ -45,8 +45,8 @@ def __init__(self,hidden_dim,sigma,gamma,tau,actor_lr,critic_lr): # initialize t self.target_critic=critic(state_dim,hidden_dim,action_dim) # create a target critic network for the agent self.actor_param=self.actor.param # parameter list of actor network, kernel uses it list for backpropagation self.critic_param=self.critic.param # parameter list of critic network, kernel uses it list for backpropagation - self.target_actor.param=self.actor_param.copy() # copy the parameters from actor network to target actor network - self.target_critic.param=self.critic_param.copy() # copy the parameters from critic network to target critic network + nn.assign_param(self.target_actor.param,self.actor_param.copy()) # copy the parameters from actor network to target actor network + nn.assign_param(self.target_critic.param,self.critic_param.copy()) # copy the parameters from critic network to target critic network self.param=[self.actor_param,self.critic_param] # parameter list of both networks, kernel uses it list for backpropagation self.sigma=sigma # noise scale self.gamma=gamma # discount factor From 2d4bc783eeb06c44aeeff9ca64254b52964033c2 Mon Sep 17 00:00:00 2001 From: NoteDance Date: Tue, 16 Jul 2024 16:34:17 +0800 Subject: [PATCH 283/337] Update DQN.py --- 7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN.py b/7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN.py index 21487674..1ea502c8 100644 --- a/7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN.py +++ b/7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN.py @@ -1,4 +1,5 @@ import tensorflow as tf # import TensorFlow library +from Note import nn import gym # import OpenAI Gym library @@ -42,5 +43,5 @@ def loss(self,s,a,next_s,r,d): # loss function, kernel uses it to calculate loss def update_param(self): # update function, kernel uses it to update parameter - self.target_q_net.param=self.param.copy() # copy the parameters from the Q-network to the target network - return \ No newline at end of file + nn.assign_param(self.target_q_net.param, self.param.copy()) # copy the parameters from the Q-network to the target network + return From d4f262b00344cf95ee56a2a46dcc4136c5cb6d75 Mon Sep 17 00:00:00 2001 From: NoteDance Date: Tue, 16 Jul 2024 16:38:05 +0800 Subject: [PATCH 284/337] Update PPO_eps.py --- 7.0/neuralnetwork/RL/tensorflow/parallel/PPO_eps.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/7.0/neuralnetwork/RL/tensorflow/parallel/PPO_eps.py b/7.0/neuralnetwork/RL/tensorflow/parallel/PPO_eps.py index 31498408..ce500fee 100644 --- a/7.0/neuralnetwork/RL/tensorflow/parallel/PPO_eps.py +++ b/7.0/neuralnetwork/RL/tensorflow/parallel/PPO_eps.py @@ -1,4 +1,5 @@ import tensorflow as tf +from Note import nn import gym from Note.nn.parallel.optimizer import SGD @@ -33,7 +34,7 @@ class PPO: def __init__(self,state_dim,hidden_dim,action_dim,clip_eps): self.actor=actor(state_dim,hidden_dim,action_dim) self.nn=actor(state_dim,hidden_dim,action_dim) - self.nn.param=self.actor.param.copy() + nn.assign(self.nn.param,self.actor.param.copy()) self.critic=critic(state_dim,hidden_dim) self.clip_eps=clip_eps self.param=[self.actor.param,self.critic.param] @@ -76,5 +77,5 @@ def opt(self,gradient): def update_param(self): - self.nn.param=self.actor.param.copy() + nn.assign(self.nn.param,self.actor.param.copy()) return From e66bdbecca84214d52f2a9a2970f9af56538d065 Mon Sep 17 00:00:00 2001 From: NoteDance Date: Tue, 16 Jul 2024 16:38:17 +0800 Subject: [PATCH 285/337] Update PPO.py --- 7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py b/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py index 66a9441f..0c9c781b 100644 --- a/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py +++ b/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py @@ -1,4 +1,5 @@ import tensorflow as tf +from Note import nn import gym from Note.nn.parallel.optimizer import SGD @@ -33,7 +34,7 @@ class PPO: def __init__(self,state_dim,hidden_dim,action_dim,clip_eps,alpha): self.actor=actor(state_dim,hidden_dim,action_dim) self.nn=actor(state_dim,hidden_dim,action_dim) - self.nn.param=self.actor.param.copy() + nn.assign(self.nn.param,self.actor.param.copy()) self.critic=critic(state_dim,hidden_dim) self.clip_eps=clip_eps self.param=[self.actor.param,self.critic.param] @@ -80,5 +81,5 @@ def opt(self,gradient): def update_param(self): - self.nn.param=self.actor.param.copy() + nn.assign(self.nn.param,self.actor.param.copy()) return From d8cb02a781fc4f689c800ec974871e1612b34f4e Mon Sep 17 00:00:00 2001 From: NoteDance Date: Tue, 16 Jul 2024 16:38:28 +0800 Subject: [PATCH 286/337] Update DQN.py --- 7.0/neuralnetwork/RL/tensorflow/parallel/DQN.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/7.0/neuralnetwork/RL/tensorflow/parallel/DQN.py b/7.0/neuralnetwork/RL/tensorflow/parallel/DQN.py index 9fa3f591..c28d47cd 100644 --- a/7.0/neuralnetwork/RL/tensorflow/parallel/DQN.py +++ b/7.0/neuralnetwork/RL/tensorflow/parallel/DQN.py @@ -1,4 +1,5 @@ import tensorflow as tf # import TensorFlow library +from Note import nn import gym # import OpenAI Gym library import Note.nn.parallel.optimizer as o # import Note's optimizer module @@ -43,10 +44,10 @@ def loss(self,s,a,next_s,r,d): # loss function, kernel uses it to calculate loss def update_param(self): # update function, kernel uses it to update parameter - self.target_q_net.param=self.param.copy() # copy the parameters from the Q-network to the target network + nn.assign(self.target_q_net.param,self.param.copy()) # copy the parameters from the Q-network to the target network return def opt(self,gradient): # optimization function, kernel uses it to optimize parameter param=self.optimizer.opt(gradient,self.param) # apply the custom momentum optimizer to update the parameters using the gradient - return param \ No newline at end of file + return param From 565e1b40c96c49a28a5e6c638914c96f3cf8fea3 Mon Sep 17 00:00:00 2001 From: NoteDance Date: Tue, 16 Jul 2024 16:38:39 +0800 Subject: [PATCH 287/337] Update DDPG.py --- 7.0/neuralnetwork/RL/tensorflow/parallel/DDPG.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/7.0/neuralnetwork/RL/tensorflow/parallel/DDPG.py b/7.0/neuralnetwork/RL/tensorflow/parallel/DDPG.py index f4c2b4ef..52401f8e 100644 --- a/7.0/neuralnetwork/RL/tensorflow/parallel/DDPG.py +++ b/7.0/neuralnetwork/RL/tensorflow/parallel/DDPG.py @@ -1,4 +1,5 @@ import tensorflow as tf # import TensorFlow library +from Note import nn import numpy as np # import NumPy library import gym # import OpenAI Gym library import Note.nn.parallel.optimizer as o # import Note's optimizer module @@ -46,8 +47,8 @@ def __init__(self,hidden_dim,sigma,gamma,tau,actor_lr,critic_lr): # initialize t self.target_critic=critic(state_dim,hidden_dim,action_dim) # create a target critic network for the agent self.actor_param=self.actor.param # parameter list of actor network, kernel uses it list for backpropagation self.critic_param=self.critic.param # parameter list of critic network, kernel uses it list for backpropagation - self.target_actor.param=self.actor_param.copy() # copy the parameters from actor network to target actor network - self.target_critic.param=self.critic_param.copy() # copy the parameters from critic network to target critic network + nn.assign_param(self.target_actor.param,self.actor_param.copy()) # copy the parameters from actor network to target actor network + nn.assign_param(self.target_critic.param,self.critic_param.copy()) # copy the parameters from critic network to target critic network self.param=[self.actor_param,self.critic_param] # parameter list of both networks, kernel uses it list for backpropagation self.sigma=sigma # noise scale self.gamma=gamma # discount factor @@ -79,7 +80,7 @@ def loss(self,s,a,next_s,r,d): # loss function, kernel uses it to calculate los def update_param(self): # update function, kernel uses it to update parameter for target_param,param in zip(self.target_actor.param,self.actor.param): # for each pair of parameters in target actor network and actor network - target_param=target_param*(1.0-self.tau)+param*self.tau # update the target parameter using soft update with factor tau + target_param.assign(target_param*(1.0-self.tau)+param*self.tau) # update the target parameter using soft update with factor tau for target_param,param in zip(self.target_critic.param,self.critic.param): # for each pair of parameters in target critic network and critic network - target_param=target_param*(1.0-self.tau)+param*self.tau # update the target parameter using soft update with factor tau - return \ No newline at end of file + target_param.assign(target_param*(1.0-self.tau)+param*self.tau) # update the target parameter using soft update with factor tau + return From 11c1c0515a504f48c59a4d71e40ea9f29d8d0d83 Mon Sep 17 00:00:00 2001 From: NoteDance Date: Tue, 16 Jul 2024 18:17:09 +0800 Subject: [PATCH 288/337] Update PPO_eps.py --- .../RL/tensorflow/non_parallel/PPO_eps.py | 24 ++++++++----------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO_eps.py b/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO_eps.py index fb00b9cd..d02f4b99 100644 --- a/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO_eps.py +++ b/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO_eps.py @@ -5,28 +5,24 @@ class actor: def __init__(self,state_dim,hidden_dim,action_dim): - self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) - self.bias1=tf.Variable(tf.random.normal([hidden_dim])) - self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) - self.bias2=tf.Variable(tf.random.normal([action_dim])) - self.param=[self.weight1,self.bias1,self.weight2,self.bias2] + self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') + self.dense2 = nn.dense(action_dim, hidden_dim) + self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list def fp(self,x): - x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) - return tf.nn.softmax(tf.matmul(x,self.weight2)+self.bias2) + x=self.dense1(x) + return tf.nn.softmax(self.dense2(x)) class critic: def __init__(self,state_dim,hidden_dim): - self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) - self.bias1=tf.Variable(tf.random.normal([hidden_dim])) - self.weight2=tf.Variable(tf.random.normal([hidden_dim,1])) - self.bias2=tf.Variable(tf.random.normal([1])) - self.param=[self.weight1,self.bias1,self.weight2,self.bias2] + self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') + self.dense2 = nn.dense(1, hidden_dim) + self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list def fp(self,x): - x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) - return tf.matmul(x,self.weight2)+self.bias2 + x=self.dense1(x) + return self.dense2(x) class PPO: From 86b8e6d93bcc93b4d5f1f7c30d1307e5372e1e56 Mon Sep 17 00:00:00 2001 From: NoteDance Date: Tue, 16 Jul 2024 18:17:19 +0800 Subject: [PATCH 289/337] Update PPO.py --- .../RL/tensorflow/non_parallel/PPO.py | 24 ++++++++----------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py b/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py index f65a524b..a4f4b088 100644 --- a/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py +++ b/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py @@ -5,28 +5,24 @@ class actor: def __init__(self,state_dim,hidden_dim,action_dim): - self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) - self.bias1=tf.Variable(tf.random.normal([hidden_dim])) - self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) - self.bias2=tf.Variable(tf.random.normal([action_dim])) - self.param=[self.weight1,self.bias1,self.weight2,self.bias2] + self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') + self.dense2 = nn.dense(action_dim, hidden_dim) + self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list def fp(self,x): - x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) - return tf.nn.softmax(tf.matmul(x,self.weight2)+self.bias2) + x=self.dense1(x) + return tf.nn.softmax(self.dense2(x)) class critic: def __init__(self,state_dim,hidden_dim): - self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) - self.bias1=tf.Variable(tf.random.normal([hidden_dim])) - self.weight2=tf.Variable(tf.random.normal([hidden_dim,1])) - self.bias2=tf.Variable(tf.random.normal([1])) - self.param=[self.weight1,self.bias1,self.weight2,self.bias2] + self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') + self.dense2 = nn.dense(1, hidden_dim) + self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list def fp(self,x): - x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) - return tf.matmul(x,self.weight2)+self.bias2 + x=self.dense1(x) + return self.dense2(x) class PPO: From 7f8dded0b7f087801aa0b2015edb851f27ba3c22 Mon Sep 17 00:00:00 2001 From: NoteDance Date: Tue, 16 Jul 2024 18:17:29 +0800 Subject: [PATCH 290/337] Update DQN_pr.py --- .../RL/tensorflow/non_parallel/DQN_pr.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN_pr.py b/7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN_pr.py index 7436ee7f..9d012933 100644 --- a/7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN_pr.py +++ b/7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN_pr.py @@ -6,15 +6,13 @@ class Qnet: def __init__(self,state_dim,hidden_dim,action_dim): - self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) - self.bias1=tf.Variable(tf.random.normal([hidden_dim])) - self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) - self.bias2=tf.Variable(tf.random.normal([action_dim])) - self.param=[self.weight1,self.bias1,self.weight2,self.bias2] + self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') + self.dense2 = nn.dense(action_dim, hidden_dim) + self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list def fp(self,x): - x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) - return tf.matmul(x,self.weight2)+self.bias2 + x = self.dense2(self.dense1(x)) + return x class DQN: From 01356a4ae61296a9c446213cf1ed746a590c64e8 Mon Sep 17 00:00:00 2001 From: NoteDance Date: Tue, 16 Jul 2024 18:17:40 +0800 Subject: [PATCH 291/337] Update DDPG.py --- .../RL/tensorflow/non_parallel/DDPG.py | 29 +++++++++---------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/7.0/neuralnetwork/RL/tensorflow/non_parallel/DDPG.py b/7.0/neuralnetwork/RL/tensorflow/non_parallel/DDPG.py index b90af114..ed07647e 100644 --- a/7.0/neuralnetwork/RL/tensorflow/non_parallel/DDPG.py +++ b/7.0/neuralnetwork/RL/tensorflow/non_parallel/DDPG.py @@ -1,36 +1,33 @@ import tensorflow as tf # import TensorFlow library +from Note import nn import numpy as np # import NumPy library import gym # import OpenAI Gym library class actor: # define a class for the actor network def __init__(self,state_dim,hidden_dim,action_dim,action_bound): # initialize the network with state dimension, hidden dimension, action dimension and action bound - self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) # create a weight matrix for the first layer - self.bias1=tf.Variable(tf.random.normal([hidden_dim])) # create a bias vector for the first layer - self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) # create a weight matrix for the second layer - self.bias2=tf.Variable(tf.random.normal([action_dim])) # create a bias vector for the second layer - self.action_bound=action_bound # store the action bound - self.param=[self.weight1,self.bias1,self.weight2,self.bias2] # store the network parameters in a list + self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') + self.dense2 = nn.dense(action_dim, hidden_dim, activation='tanh') + self.action_bound=action_bound + self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list def fp(self,x): # forward propagation function, kernel uses it for forward propagation - x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) # apply the first layer with ReLU activation - return tf.nn.tanh(tf.matmul(x,self.weight2)+self.bias2)*self.action_bound # apply the second layer with tanh activation and scale it by the action bound + x = self.dense1(x) + return self.dense2(x)*self.action_bound class critic: # define a class for the critic network def __init__(self,state_dim,hidden_dim,action_dim): # initialize the network with state dimension, hidden dimension and action dimension - self.weight1=tf.Variable(tf.random.normal([state_dim+action_dim,hidden_dim])) # create a weight matrix for the first layer - self.bias1=tf.Variable(tf.random.normal([hidden_dim])) # create a bias vector for the first layer - self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) # create a weight matrix for the second layer - self.bias2=tf.Variable(tf.random.normal([action_dim])) # create a bias vector for the second layer - self.param=[self.weight1,self.bias1,self.weight2,self.bias2] # store the network parameters in a list + self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') + self.dense2 = nn.dense(action_dim, hidden_dim) + self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list def fp(self,x,a): # forward propagation function, kernel uses it for forward propagation - cat=tf.concat([x,a],axis=1) # concatenate the state and action vectors along the first axis - x=tf.nn.relu(tf.matmul(cat,self.weight1)+self.bias1) # apply the first layer with ReLU activation - return tf.matmul(x,self.weight2)+self.bias2 # apply the second layer and return the output + cat=tf.concat([x,a],axis=1) + x=self.dense1(cat) + return self.dense2(x) class DDPG: # define a class for the DDPG agent From ec893be0aa5c9c610e790200c2b3476e4e306f72 Mon Sep 17 00:00:00 2001 From: NoteDance Date: Tue, 16 Jul 2024 18:17:50 +0800 Subject: [PATCH 292/337] Update DQN.py --- 7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN.py b/7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN.py index 1ea502c8..b3df75e1 100644 --- a/7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN.py +++ b/7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN.py @@ -5,15 +5,13 @@ class Qnet: # define a class for the Q-network def __init__(self,state_dim,hidden_dim,action_dim): # initialize the network with state dimension, hidden dimension and action dimension - self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) # create a weight matrix for the first layer - self.bias1=tf.Variable(tf.random.normal([hidden_dim])) # create a bias vector for the first layer - self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) # create a weight matrix for the second layer - self.bias2=tf.Variable(tf.random.normal([action_dim])) # create a bias vector for the second layer - self.param=[self.weight1,self.bias1,self.weight2,self.bias2] # store the network parameters in a list + self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') + self.dense2 = nn.dense(action_dim, hidden_dim) + self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list def fp(self,x): # forward propagation function, kernel uses it for forward propagation - x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) # apply the first layer with ReLU activation - return tf.matmul(x,self.weight2)+self.bias2 # apply the second layer and return the output + x = self.dense2(self.dense1(x)) + return x class DQN: # define a class for the DQN agent From 7101f55bef9afa83e13dfb7fa2e8259061c0f2ed Mon Sep 17 00:00:00 2001 From: NoteDance Date: Tue, 16 Jul 2024 18:20:25 +0800 Subject: [PATCH 293/337] Update DDPG.py --- .../RL/tensorflow/parallel/DDPG.py | 28 ++++++++----------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/7.0/neuralnetwork/RL/tensorflow/parallel/DDPG.py b/7.0/neuralnetwork/RL/tensorflow/parallel/DDPG.py index 52401f8e..90b4caa1 100644 --- a/7.0/neuralnetwork/RL/tensorflow/parallel/DDPG.py +++ b/7.0/neuralnetwork/RL/tensorflow/parallel/DDPG.py @@ -7,32 +7,28 @@ class actor: # define a class for the actor network def __init__(self,state_dim,hidden_dim,action_dim,action_bound): # initialize the network with state dimension, hidden dimension, action dimension and action bound - self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) # create a weight matrix for the first layer - self.bias1=tf.Variable(tf.random.normal([hidden_dim])) # create a bias vector for the first layer - self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) # create a weight matrix for the second layer - self.bias2=tf.Variable(tf.random.normal([action_dim])) # create a bias vector for the second layer - self.action_bound=action_bound # store the action bound - self.param=[self.weight1,self.bias1,self.weight2,self.bias2] # store the network parameters in a list + self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') + self.dense2 = nn.dense(action_dim, hidden_dim, activation='tanh') + self.action_bound=action_bound + self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list def fp(self,x): # forward propagation function, kernel uses it for forward propagation - x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) # apply the first layer with ReLU activation - return tf.nn.tanh(tf.matmul(x,self.weight2)+self.bias2)*self.action_bound # apply the second layer with tanh activation and scale it by the action bound + x = self.dense1(x) + return self.dense2(x)*self.action_bound class critic: # define a class for the critic network def __init__(self,state_dim,hidden_dim,action_dim): # initialize the network with state dimension, hidden dimension and action dimension - self.weight1=tf.Variable(tf.random.normal([state_dim+action_dim,hidden_dim])) # create a weight matrix for the first layer - self.bias1=tf.Variable(tf.random.normal([hidden_dim])) # create a bias vector for the first layer - self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) # create a weight matrix for the second layer - self.bias2=tf.Variable(tf.random.normal([action_dim])) # create a bias vector for the second layer - self.param=[self.weight1,self.bias1,self.weight2,self.bias2] # store the network parameters in a list + self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') + self.dense2 = nn.dense(action_dim, hidden_dim) + self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list def fp(self,x,a): # forward propagation function, kernel uses it for forward propagation - cat=tf.concat([x,a],axis=1) # concatenate the state and action vectors along the first axis - x=tf.nn.relu(tf.matmul(cat,self.weight1)+self.bias1) # apply the first layer with ReLU activation - return tf.matmul(x,self.weight2)+self.bias2 # apply the second layer and return the output + cat=tf.concat([x,a],axis=1) + x=self.dense1(cat) + return self.dense2(x) class DDPG: # define a class for the DDPG agent From 2223686dcae9fb6c9f8865fe9270a33a290377fe Mon Sep 17 00:00:00 2001 From: NoteDance Date: Tue, 16 Jul 2024 18:20:51 +0800 Subject: [PATCH 294/337] Update DQN.py --- 7.0/neuralnetwork/RL/tensorflow/parallel/DQN.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/7.0/neuralnetwork/RL/tensorflow/parallel/DQN.py b/7.0/neuralnetwork/RL/tensorflow/parallel/DQN.py index c28d47cd..f9a0b3fb 100644 --- a/7.0/neuralnetwork/RL/tensorflow/parallel/DQN.py +++ b/7.0/neuralnetwork/RL/tensorflow/parallel/DQN.py @@ -6,15 +6,13 @@ class Qnet: # define a class for the Q-network def __init__(self,state_dim,hidden_dim,action_dim): # initialize the network with state dimension, hidden dimension and action dimension - self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) # create a weight matrix for the first layer - self.bias1=tf.Variable(tf.random.normal([hidden_dim])) # create a bias vector for the first layer - self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) # create a weight matrix for the second layer - self.bias2=tf.Variable(tf.random.normal([action_dim])) # create a bias vector for the second layer - self.param=[self.weight1,self.bias1,self.weight2,self.bias2] # store the network parameters in a list + self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') + self.dense2 = nn.dense(action_dim, hidden_dim) + self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list def fp(self,x): # forward propagation function, kernel uses it for forward propagation - x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) # apply the first layer with ReLU activation - return tf.matmul(x,self.weight2)+self.bias2 # apply the second layer and return the output + x = self.dense2(self.dense1(x)) + return x class DQN: # define a class for the DQN agent From 7622dbe6dd906523d17090e7ae8b801ebd1b961b Mon Sep 17 00:00:00 2001 From: NoteDance Date: Tue, 16 Jul 2024 18:21:15 +0800 Subject: [PATCH 295/337] Update PPO.py --- .../RL/tensorflow/parallel/PPO.py | 26 ++++++++----------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py b/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py index 0c9c781b..d76df0cf 100644 --- a/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py +++ b/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py @@ -6,28 +6,24 @@ class actor: def __init__(self,state_dim,hidden_dim,action_dim): - self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) - self.bias1=tf.Variable(tf.random.normal([hidden_dim])) - self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) - self.bias2=tf.Variable(tf.random.normal([action_dim])) - self.param=[self.weight1,self.bias1,self.weight2,self.bias2] + self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') + self.dense2 = nn.dense(action_dim, hidden_dim) + self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list def fp(self,x): - x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) - return tf.nn.softmax(tf.matmul(x,self.weight2)+self.bias2) + x=self.dense1(x) + return tf.nn.softmax(self.dense2(x)) class critic: - def __init__(self,state_dim,hidden_dim,action_dim): - self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) - self.bias1=tf.Variable(tf.random.normal([hidden_dim])) - self.weight2=tf.Variable(tf.random.normal([hidden_dim,1])) - self.bias2=tf.Variable(tf.random.normal([1])) - self.param=[self.weight1,self.bias1,self.weight2,self.bias2] + def __init__(self,state_dim,hidden_dim): + self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') + self.dense2 = nn.dense(1, hidden_dim) + self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list def fp(self,x): - x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) - return tf.matmul(x,self.weight2)+self.bias2 + x=self.dense1(x) + return self.dense2(x) class PPO: From 6a2e034078c40d796e52a8cf1d09ae614d3fbf58 Mon Sep 17 00:00:00 2001 From: NoteDance Date: Tue, 16 Jul 2024 18:21:41 +0800 Subject: [PATCH 296/337] Update PPO_eps.py --- .../RL/tensorflow/parallel/PPO_eps.py | 26 ++++++++----------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/7.0/neuralnetwork/RL/tensorflow/parallel/PPO_eps.py b/7.0/neuralnetwork/RL/tensorflow/parallel/PPO_eps.py index ce500fee..1c9e0369 100644 --- a/7.0/neuralnetwork/RL/tensorflow/parallel/PPO_eps.py +++ b/7.0/neuralnetwork/RL/tensorflow/parallel/PPO_eps.py @@ -6,28 +6,24 @@ class actor: def __init__(self,state_dim,hidden_dim,action_dim): - self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) - self.bias1=tf.Variable(tf.random.normal([hidden_dim])) - self.weight2=tf.Variable(tf.random.normal([hidden_dim,action_dim])) - self.bias2=tf.Variable(tf.random.normal([action_dim])) - self.param=[self.weight1,self.bias1,self.weight2,self.bias2] + self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') + self.dense2 = nn.dense(action_dim, hidden_dim) + self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list def fp(self,x): - x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) - return tf.nn.softmax(tf.matmul(x,self.weight2)+self.bias2) + x=self.dense1(x) + return tf.nn.softmax(self.dense2(x)) class critic: - def __init__(self,state_dim,hidden_dim,action_dim): - self.weight1=tf.Variable(tf.random.normal([state_dim,hidden_dim])) - self.bias1=tf.Variable(tf.random.normal([hidden_dim])) - self.weight2=tf.Variable(tf.random.normal([hidden_dim,1])) - self.bias2=tf.Variable(tf.random.normal([1])) - self.param=[self.weight1,self.bias1,self.weight2,self.bias2] + def __init__(self,state_dim,hidden_dim): + self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') + self.dense2 = nn.dense(1, hidden_dim) + self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list def fp(self,x): - x=tf.nn.relu(tf.matmul(x,self.weight1)+self.bias1) - return tf.matmul(x,self.weight2)+self.bias2 + x=self.dense1(x) + return self.dense2(x) class PPO: From f9a75ac87b3b72daeee9a2e52e88e9c483334b11 Mon Sep 17 00:00:00 2001 From: NoteDance Date: Tue, 16 Jul 2024 18:39:44 +0800 Subject: [PATCH 297/337] Update DQN_pr.py --- 7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN_pr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN_pr.py b/7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN_pr.py index 9d012933..d9b03d46 100644 --- a/7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN_pr.py +++ b/7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN_pr.py @@ -1,7 +1,7 @@ import tensorflow as tf from Note import nn import gym -import Note.create.RL.rl.prioritized_replay as pr +import Note.RL.rl.prioritized_replay as pr class Qnet: From e47e60cf63ad6d7579a21e32f6a2238b3aad1e44 Mon Sep 17 00:00:00 2001 From: NoteDance Date: Tue, 16 Jul 2024 18:57:22 +0800 Subject: [PATCH 298/337] Update DQN_pr.py --- 7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN_pr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN_pr.py b/7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN_pr.py index d9b03d46..41a1cd2e 100644 --- a/7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN_pr.py +++ b/7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN_pr.py @@ -22,7 +22,7 @@ def __init__(self,state_dim,hidden_dim,action_dim): self.param=self.nn.param self.pr=pr.pr() self.initial_TD=7 - self._epsilon=0.0007 + self.epsilon=0.0007 self.alpha=0.7 self.opt=tf.keras.optimizers.Adam() self.genv=gym.make('CartPole-v0') From 83d4946f8b4245dfec57993293448bb7d2b37a1a Mon Sep 17 00:00:00 2001 From: NoteDance Date: Wed, 17 Jul 2024 00:55:52 +0800 Subject: [PATCH 299/337] Update DDPG.py --- 7.0/neuralnetwork/RL/tensorflow/parallel/DDPG.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/7.0/neuralnetwork/RL/tensorflow/parallel/DDPG.py b/7.0/neuralnetwork/RL/tensorflow/parallel/DDPG.py index 90b4caa1..713917d8 100644 --- a/7.0/neuralnetwork/RL/tensorflow/parallel/DDPG.py +++ b/7.0/neuralnetwork/RL/tensorflow/parallel/DDPG.py @@ -20,7 +20,7 @@ def fp(self,x): # forward propagation function, kernel uses it for forward prop class critic: # define a class for the critic network def __init__(self,state_dim,hidden_dim,action_dim): # initialize the network with state dimension, hidden dimension and action dimension - self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') + self.dense1 = nn.dense(hidden_dim, state_dim+1, activation='relu') self.dense2 = nn.dense(action_dim, hidden_dim) self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list @@ -33,7 +33,7 @@ def fp(self,x,a): # forward propagation function, kernel uses it for forward pr class DDPG: # define a class for the DDPG agent def __init__(self,hidden_dim,sigma,gamma,tau,actor_lr,critic_lr): # initialize the agent with hidden dimension, noise scale, discount factor, soft update factor, actor learning rate and critic learning rate - self.genv=[gym.make('Pendulum-v0') for _ in range(5)] # create a list of 5 environments + self.genv=[gym.make('Pendulum-v1') for _ in range(5)] # create a list of 5 environments state_dim=self.genv.observation_space.shape[0] # get the state dimension from the environment observation space action_dim=self.genv.action_space.shape[0] # get the action dimension from the environment action space action_bound=self.genv.action_space.high[0] # get the action bound from the environment action space high value (assume symmetric) From 4c02911a50e1c5870c8d308930250039905bba6e Mon Sep 17 00:00:00 2001 From: NoteDance Date: Wed, 17 Jul 2024 00:56:06 +0800 Subject: [PATCH 300/337] Update DDPG.py --- 7.0/neuralnetwork/RL/tensorflow/non_parallel/DDPG.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/7.0/neuralnetwork/RL/tensorflow/non_parallel/DDPG.py b/7.0/neuralnetwork/RL/tensorflow/non_parallel/DDPG.py index ed07647e..dd12aa45 100644 --- a/7.0/neuralnetwork/RL/tensorflow/non_parallel/DDPG.py +++ b/7.0/neuralnetwork/RL/tensorflow/non_parallel/DDPG.py @@ -19,7 +19,7 @@ def fp(self,x): # forward propagation function, kernel uses it for forward prop class critic: # define a class for the critic network def __init__(self,state_dim,hidden_dim,action_dim): # initialize the network with state dimension, hidden dimension and action dimension - self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') + self.dense1 = nn.dense(hidden_dim, state_dim+1, activation='relu') self.dense2 = nn.dense(action_dim, hidden_dim) self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list @@ -32,7 +32,7 @@ def fp(self,x,a): # forward propagation function, kernel uses it for forward pr class DDPG: # define a class for the DDPG agent def __init__(self,hidden_dim,sigma,gamma,tau,actor_lr,critic_lr): # initialize the agent with hidden dimension, noise scale, discount factor, soft update factor, actor learning rate and critic learning rate - self.genv=gym.make('Pendulum-v0') # create environment + self.genv=gym.make('Pendulum-v1') # create environment state_dim=self.genv.observation_space.shape[0] # get the state dimension from the environment observation space action_dim=self.genv.action_space.shape[0] # get the action dimension from the environment action space action_bound=self.genv.action_space.high[0] # get the action bound from the environment action space high value (assume symmetric) From bfa7cea0fc7eb337e2226acfec4ca09c394e2d42 Mon Sep 17 00:00:00 2001 From: NoteDance Date: Wed, 17 Jul 2024 00:56:56 +0800 Subject: [PATCH 301/337] Update DDPG.py --- 7.0/neuralnetwork/RL/pytorch/non_parallel/DDPG.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/7.0/neuralnetwork/RL/pytorch/non_parallel/DDPG.py b/7.0/neuralnetwork/RL/pytorch/non_parallel/DDPG.py index 4b0abd94..3a6cdb28 100644 --- a/7.0/neuralnetwork/RL/pytorch/non_parallel/DDPG.py +++ b/7.0/neuralnetwork/RL/pytorch/non_parallel/DDPG.py @@ -36,7 +36,7 @@ def __init__(self,hidden_dim,sigma,gamma,tau,actor_lr,critic_lr): self.device=torch.device('cuda') else: self.device=torch.device('cpu') - self.genv=gym.make('Pendulum-v0') + self.genv=gym.make('Pendulum-v1') state_dim=self.genv.observation_space.shape[0] action_dim=self.genv.action_space.shape[0] action_bound=self.genv.action_space.high[0] @@ -98,4 +98,4 @@ def update_param(self): target_param.data.copy_(target_param.data*(1.0-self.tau)+param.data*self.tau) for target_param,param in zip(self.target_critic.parameters(),self.critic.parameters()): target_param.data.copy_(target_param.data*(1.0-self.tau)+param.data*self.tau) - return \ No newline at end of file + return From 42ef7ec5893c85eeca47033b0bb8692b7273c2b0 Mon Sep 17 00:00:00 2001 From: NoteDance Date: Wed, 17 Jul 2024 00:57:49 +0800 Subject: [PATCH 302/337] Update DDPG.py --- 7.0/neuralnetwork/RL/tensorflow/parallel/DDPG.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/7.0/neuralnetwork/RL/tensorflow/parallel/DDPG.py b/7.0/neuralnetwork/RL/tensorflow/parallel/DDPG.py index 713917d8..bb071ff2 100644 --- a/7.0/neuralnetwork/RL/tensorflow/parallel/DDPG.py +++ b/7.0/neuralnetwork/RL/tensorflow/parallel/DDPG.py @@ -20,7 +20,7 @@ def fp(self,x): # forward propagation function, kernel uses it for forward prop class critic: # define a class for the critic network def __init__(self,state_dim,hidden_dim,action_dim): # initialize the network with state dimension, hidden dimension and action dimension - self.dense1 = nn.dense(hidden_dim, state_dim+1, activation='relu') + self.dense1 = nn.dense(hidden_dim, state_dim+action_dim, activation='relu') self.dense2 = nn.dense(action_dim, hidden_dim) self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list From 4e3806f63cbf6f844eb306ece1d5ab15a8cda997 Mon Sep 17 00:00:00 2001 From: NoteDance Date: Wed, 17 Jul 2024 00:58:02 +0800 Subject: [PATCH 303/337] Update DDPG.py --- 7.0/neuralnetwork/RL/tensorflow/non_parallel/DDPG.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/7.0/neuralnetwork/RL/tensorflow/non_parallel/DDPG.py b/7.0/neuralnetwork/RL/tensorflow/non_parallel/DDPG.py index dd12aa45..9626fa2d 100644 --- a/7.0/neuralnetwork/RL/tensorflow/non_parallel/DDPG.py +++ b/7.0/neuralnetwork/RL/tensorflow/non_parallel/DDPG.py @@ -19,7 +19,7 @@ def fp(self,x): # forward propagation function, kernel uses it for forward prop class critic: # define a class for the critic network def __init__(self,state_dim,hidden_dim,action_dim): # initialize the network with state dimension, hidden dimension and action dimension - self.dense1 = nn.dense(hidden_dim, state_dim+1, activation='relu') + self.dense1 = nn.dense(hidden_dim, state_dim+action_dim, activation='relu') self.dense2 = nn.dense(action_dim, hidden_dim) self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list From 405a0e61066514512d5cbf1bfdbf30daa6511937 Mon Sep 17 00:00:00 2001 From: NoteDance Date: Wed, 17 Jul 2024 14:19:09 +0800 Subject: [PATCH 304/337] Add files via upload --- 7.0/neuralnetwork/RL/tensorflow/non_parallel/DDPG.py | 4 ++-- 7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN.py | 4 ++-- 7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN_pr.py | 4 ++-- 7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py | 4 ++-- 7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO_eps.py | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/7.0/neuralnetwork/RL/tensorflow/non_parallel/DDPG.py b/7.0/neuralnetwork/RL/tensorflow/non_parallel/DDPG.py index 9626fa2d..cb028d88 100644 --- a/7.0/neuralnetwork/RL/tensorflow/non_parallel/DDPG.py +++ b/7.0/neuralnetwork/RL/tensorflow/non_parallel/DDPG.py @@ -67,7 +67,7 @@ def env(self,a=None,initial=None): # environment function, kernel uses it to in def loss(self,s,a,next_s,r,d): # loss function, kernel uses it to calculate loss a=tf.expand_dims(a,axis=1) # expand the action vector to match the Q-value matrix shape next_q_value=self.target_critic.fp(next_s,self.target_actor.fp(next_s)) # get the Q-value for the next state and action from the target critic network - q_target=r+self.gamma*next_q_value*(1-d) # calculate the target value using Bellman equation with discount factor gamma + q_target=r+self.gamma*next_q_value*(1-tf.cast(d,'float32')) # calculate the target value using Bellman equation with discount factor gamma actor_loss=-tf.reduce_mean(self.critic.fp(s,self.actor.fp(s))) # calculate the actor loss as the negative mean of the Q-value for the current state and action from the critic network critic_loss=tf.reduce_mean((self.critic.fp(s,a)-q_target)**2) # calculate the critic loss as the mean squared error between Q-value and target value return [actor_loss,critic_loss] # return a list of actor loss and critic loss @@ -78,4 +78,4 @@ def update_param(self): # update function, kernel uses it to update parameter target_param.assign(target_param*(1.0-self.tau)+param*self.tau) # update the target parameter using soft update with factor tau for target_param,param in zip(self.target_critic.param,self.critic.param): # for each pair of parameters in target critic network and critic network target_param.assign(target_param*(1.0-self.tau)+param*self.tau) # update the target parameter using soft update with factor tau - return + return \ No newline at end of file diff --git a/7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN.py b/7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN.py index b3df75e1..b8ea6f1c 100644 --- a/7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN.py +++ b/7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN.py @@ -36,10 +36,10 @@ def loss(self,s,a,next_s,r,d): # loss function, kernel uses it to calculate loss a=tf.expand_dims(a,axis=1) # expand the action vector to match the Q-value matrix shape q_value=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) # get the Q-value for the selected action next_q_value=tf.reduce_max(self.target_q_net.fp(next_s),axis=1) # get the maximum Q-value for the next state from the target network - target=r+0.98*next_q_value*(1-d) # calculate the target value using Bellman equation with discount factor 0.98 + target=r+0.98*next_q_value*(1-tf.cast(d,'float32')) # calculate the target value using Bellman equation with discount factor 0.98 return tf.reduce_mean((q_value-target)**2) # return the mean squared error between Q-value and target value def update_param(self): # update function, kernel uses it to update parameter nn.assign_param(self.target_q_net.param, self.param.copy()) # copy the parameters from the Q-network to the target network - return + return \ No newline at end of file diff --git a/7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN_pr.py b/7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN_pr.py index 41a1cd2e..1f901d5d 100644 --- a/7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN_pr.py +++ b/7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN_pr.py @@ -46,7 +46,7 @@ def loss(self,s,a,next_s,r,d): a=tf.expand_dims(a,axis=1) q_value=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) next_q_value=tf.reduce_max(self.target_q_net.fp(next_s),axis=1) - target=r+0.98*next_q_value*(1-d) + target=r+0.98*next_q_value*(1-tf.cast(d,'float32')) target=tf.expand_dims(target,axis=1) TD=target-q_value self.pr.update_TD(TD) @@ -55,4 +55,4 @@ def loss(self,s,a,next_s,r,d): def update_param(self): nn.assign_param(self.target_q_net.param, self.param.copy()) - return + return \ No newline at end of file diff --git a/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py b/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py index a4f4b088..092fc7c4 100644 --- a/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py +++ b/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py @@ -53,7 +53,7 @@ def loss(self,s,a,next_s,r,d): action_prob_old=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) raito=action_prob/action_prob_old value=self.critic.fp(s) - value_tar=r+0.98*self.critic.fp(next_s)*(1-d) + value_tar=r+0.98*self.critic.fp(next_s)*(1-tf.cast(d,'float32')) TD=value_tar-value sur1=raito*TD sur2=tf.clip_by_value(raito,clip_value_min=1-self.clip_eps,clip_value_max=1+self.clip_eps)*TD @@ -79,4 +79,4 @@ def opt(self,gradient): def update_param(self): nn.assign_param(self.nn.param, self.actor.param.copy()) - return + return \ No newline at end of file diff --git a/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO_eps.py b/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO_eps.py index d02f4b99..947cea51 100644 --- a/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO_eps.py +++ b/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO_eps.py @@ -50,7 +50,7 @@ def loss(self,s,a,next_s,r,d): a=tf.expand_dims(a,axis=1) raito=tf.gather(self.actor.fp(s),a,axis=1,batch_dims=1)/tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) value=self.critic.fp(s) - value_tar=r+0.98*self.critic.fp(next_s)*(1-d) + value_tar=r+0.98*self.critic.fp(next_s)*(1-tf.cast(d,'float32')) TD=value_tar-value sur1=raito*TD sur2=tf.clip_by_value(raito,clip_value_min=1-self.clip_eps,clip_value_max=1+self.clip_eps)*TD @@ -73,4 +73,4 @@ def opt(self,gradient): def update_param(self): nn.assign_param(self.nn.param, self.actor.param.copy()) - return + return \ No newline at end of file From 6204c4c8a8965d0df960c82c7e305ee35a99c8a9 Mon Sep 17 00:00:00 2001 From: NoteDance Date: Wed, 17 Jul 2024 14:20:11 +0800 Subject: [PATCH 305/337] Add files via upload --- 7.0/neuralnetwork/RL/tensorflow/parallel/DDPG.py | 4 ++-- 7.0/neuralnetwork/RL/tensorflow/parallel/DQN.py | 4 ++-- 7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py | 4 ++-- 7.0/neuralnetwork/RL/tensorflow/parallel/PPO_eps.py | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/7.0/neuralnetwork/RL/tensorflow/parallel/DDPG.py b/7.0/neuralnetwork/RL/tensorflow/parallel/DDPG.py index bb071ff2..0f297711 100644 --- a/7.0/neuralnetwork/RL/tensorflow/parallel/DDPG.py +++ b/7.0/neuralnetwork/RL/tensorflow/parallel/DDPG.py @@ -68,7 +68,7 @@ def env(self,a=None,p=None,initial=None): # environment function, kernel uses it def loss(self,s,a,next_s,r,d): # loss function, kernel uses it to calculate loss a=tf.expand_dims(a,axis=1) # expand the action vector to match the Q-value matrix shape next_q_value=self.target_critic.fp(next_s,self.target_actor.fp(next_s)) # get the Q-value for the next state and action from the target critic network - q_target=r+self.gamma*next_q_value*(1-d) # calculate the target value using Bellman equation with discount factor gamma + q_target=r+self.gamma*next_q_value*(1-tf.cast(d,'float32')) # calculate the target value using Bellman equation with discount factor gamma actor_loss=-tf.reduce_mean(self.critic.fp(s,self.actor.fp(s))) # calculate the actor loss as the negative mean of the Q-value for the current state and action from the critic network critic_loss=tf.reduce_mean((self.critic.fp(s,a)-q_target)**2) # calculate the critic loss as the mean squared error between Q-value and target value return [actor_loss,critic_loss] # return a list of actor loss and critic loss @@ -79,4 +79,4 @@ def update_param(self): # update function, kernel uses it to update parameter target_param.assign(target_param*(1.0-self.tau)+param*self.tau) # update the target parameter using soft update with factor tau for target_param,param in zip(self.target_critic.param,self.critic.param): # for each pair of parameters in target critic network and critic network target_param.assign(target_param*(1.0-self.tau)+param*self.tau) # update the target parameter using soft update with factor tau - return + return \ No newline at end of file diff --git a/7.0/neuralnetwork/RL/tensorflow/parallel/DQN.py b/7.0/neuralnetwork/RL/tensorflow/parallel/DQN.py index f9a0b3fb..d75c02bc 100644 --- a/7.0/neuralnetwork/RL/tensorflow/parallel/DQN.py +++ b/7.0/neuralnetwork/RL/tensorflow/parallel/DQN.py @@ -37,7 +37,7 @@ def loss(self,s,a,next_s,r,d): # loss function, kernel uses it to calculate loss a=tf.expand_dims(a,axis=1) # expand the action vector to match the Q-value matrix shape q_value=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) # get the Q-value for the selected action next_q_value=tf.reduce_max(self.target_q_net.fp(next_s),axis=1) # get the maximum Q-value for the next state from the target network - target=r+0.98*next_q_value*(1-d) # calculate the target value using Bellman equation with discount factor 0.98 + target=r+0.98*next_q_value*(1-tf.cast(d,'float32')) # calculate the target value using Bellman equation with discount factor 0.98 return tf.reduce_mean((q_value-target)**2) # return the mean squared error between Q-value and target value @@ -48,4 +48,4 @@ def update_param(self): # update function, kernel uses it to update parameter def opt(self,gradient): # optimization function, kernel uses it to optimize parameter param=self.optimizer.opt(gradient,self.param) # apply the custom momentum optimizer to update the parameters using the gradient - return param + return param \ No newline at end of file diff --git a/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py b/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py index d76df0cf..67c59647 100644 --- a/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py +++ b/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py @@ -53,7 +53,7 @@ def loss(self,s,a,next_s,r,d): action_prob_old=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) raito=action_prob/action_prob_old value=self.critic.fp(s) - value_tar=r+0.98*self.critic.fp(next_s)*(1-d) + value_tar=r+0.98*self.critic.fp(next_s)*(1-tf.cast(d,'float32')) TD=value_tar-value sur1=raito*TD sur2=tf.clip_by_value(raito,clip_value_min=1-self.clip_eps,clip_value_max=1+self.clip_eps)*TD @@ -78,4 +78,4 @@ def opt(self,gradient): def update_param(self): nn.assign(self.nn.param,self.actor.param.copy()) - return + return \ No newline at end of file diff --git a/7.0/neuralnetwork/RL/tensorflow/parallel/PPO_eps.py b/7.0/neuralnetwork/RL/tensorflow/parallel/PPO_eps.py index 1c9e0369..39229ee4 100644 --- a/7.0/neuralnetwork/RL/tensorflow/parallel/PPO_eps.py +++ b/7.0/neuralnetwork/RL/tensorflow/parallel/PPO_eps.py @@ -51,7 +51,7 @@ def loss(self,s,a,next_s,r,d): a=tf.expand_dims(a,axis=1) raito=tf.gather(self.actor.fp(s),a,axis=1,batch_dims=1)/tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) value=self.critic.fp(s) - value_tar=r+0.98*self.critic.fp(next_s)*(1-d) + value_tar=r+0.98*self.critic.fp(next_s)*(1-tf.cast(d,'float32')) TD=value_tar-value sur1=raito*TD sur2=tf.clip_by_value(raito,clip_value_min=1-self.clip_eps,clip_value_max=1+self.clip_eps)*TD @@ -74,4 +74,4 @@ def opt(self,gradient): def update_param(self): nn.assign(self.nn.param,self.actor.param.copy()) - return + return \ No newline at end of file From 2d35b35f8ab3b1f0fd5423610760ca9cce1ed0cc Mon Sep 17 00:00:00 2001 From: NoteDance Date: Wed, 17 Jul 2024 16:59:27 +0800 Subject: [PATCH 306/337] Add files via upload --- 7.0/neuralnetwork/RL/tensorflow/parallel/DDPG.py | 2 +- 7.0/neuralnetwork/RL/tensorflow/parallel/DQN.py | 2 +- 7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py | 2 +- 7.0/neuralnetwork/RL/tensorflow/parallel/PPO_eps.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/7.0/neuralnetwork/RL/tensorflow/parallel/DDPG.py b/7.0/neuralnetwork/RL/tensorflow/parallel/DDPG.py index 0f297711..35084078 100644 --- a/7.0/neuralnetwork/RL/tensorflow/parallel/DDPG.py +++ b/7.0/neuralnetwork/RL/tensorflow/parallel/DDPG.py @@ -68,7 +68,7 @@ def env(self,a=None,p=None,initial=None): # environment function, kernel uses it def loss(self,s,a,next_s,r,d): # loss function, kernel uses it to calculate loss a=tf.expand_dims(a,axis=1) # expand the action vector to match the Q-value matrix shape next_q_value=self.target_critic.fp(next_s,self.target_actor.fp(next_s)) # get the Q-value for the next state and action from the target critic network - q_target=r+self.gamma*next_q_value*(1-tf.cast(d,'float32')) # calculate the target value using Bellman equation with discount factor gamma + q_target=tf.cast(r,'float32')+self.gamma*next_q_value*(1-tf.cast(d,'float32')) # calculate the target value using Bellman equation with discount factor gamma actor_loss=-tf.reduce_mean(self.critic.fp(s,self.actor.fp(s))) # calculate the actor loss as the negative mean of the Q-value for the current state and action from the critic network critic_loss=tf.reduce_mean((self.critic.fp(s,a)-q_target)**2) # calculate the critic loss as the mean squared error between Q-value and target value return [actor_loss,critic_loss] # return a list of actor loss and critic loss diff --git a/7.0/neuralnetwork/RL/tensorflow/parallel/DQN.py b/7.0/neuralnetwork/RL/tensorflow/parallel/DQN.py index d75c02bc..889e7b03 100644 --- a/7.0/neuralnetwork/RL/tensorflow/parallel/DQN.py +++ b/7.0/neuralnetwork/RL/tensorflow/parallel/DQN.py @@ -37,7 +37,7 @@ def loss(self,s,a,next_s,r,d): # loss function, kernel uses it to calculate loss a=tf.expand_dims(a,axis=1) # expand the action vector to match the Q-value matrix shape q_value=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) # get the Q-value for the selected action next_q_value=tf.reduce_max(self.target_q_net.fp(next_s),axis=1) # get the maximum Q-value for the next state from the target network - target=r+0.98*next_q_value*(1-tf.cast(d,'float32')) # calculate the target value using Bellman equation with discount factor 0.98 + target=tf.cast(r,'float32')+0.98*next_q_value*(1-tf.cast(d,'float32')) # calculate the target value using Bellman equation with discount factor 0.98 return tf.reduce_mean((q_value-target)**2) # return the mean squared error between Q-value and target value diff --git a/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py b/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py index 67c59647..29923b2b 100644 --- a/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py +++ b/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py @@ -53,7 +53,7 @@ def loss(self,s,a,next_s,r,d): action_prob_old=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) raito=action_prob/action_prob_old value=self.critic.fp(s) - value_tar=r+0.98*self.critic.fp(next_s)*(1-tf.cast(d,'float32')) + value_tar=tf.cast(r,'float32')+0.98*self.critic.fp(next_s)*(1-tf.cast(d,'float32')) TD=value_tar-value sur1=raito*TD sur2=tf.clip_by_value(raito,clip_value_min=1-self.clip_eps,clip_value_max=1+self.clip_eps)*TD diff --git a/7.0/neuralnetwork/RL/tensorflow/parallel/PPO_eps.py b/7.0/neuralnetwork/RL/tensorflow/parallel/PPO_eps.py index 39229ee4..3161a91a 100644 --- a/7.0/neuralnetwork/RL/tensorflow/parallel/PPO_eps.py +++ b/7.0/neuralnetwork/RL/tensorflow/parallel/PPO_eps.py @@ -51,7 +51,7 @@ def loss(self,s,a,next_s,r,d): a=tf.expand_dims(a,axis=1) raito=tf.gather(self.actor.fp(s),a,axis=1,batch_dims=1)/tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) value=self.critic.fp(s) - value_tar=r+0.98*self.critic.fp(next_s)*(1-tf.cast(d,'float32')) + value_tar=tf.cast(r,'float32')+0.98*self.critic.fp(next_s)*(1-tf.cast(d,'float32')) TD=value_tar-value sur1=raito*TD sur2=tf.clip_by_value(raito,clip_value_min=1-self.clip_eps,clip_value_max=1+self.clip_eps)*TD From a013565f120055d323df30ad7ece26af57d96602 Mon Sep 17 00:00:00 2001 From: NoteDance Date: Wed, 17 Jul 2024 17:00:22 +0800 Subject: [PATCH 307/337] Add files via upload --- 7.0/neuralnetwork/RL/tensorflow/non_parallel/DDPG.py | 2 +- 7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN.py | 2 +- 7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN_pr.py | 2 +- 7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py | 2 +- 7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO_eps.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/7.0/neuralnetwork/RL/tensorflow/non_parallel/DDPG.py b/7.0/neuralnetwork/RL/tensorflow/non_parallel/DDPG.py index cb028d88..47c59e65 100644 --- a/7.0/neuralnetwork/RL/tensorflow/non_parallel/DDPG.py +++ b/7.0/neuralnetwork/RL/tensorflow/non_parallel/DDPG.py @@ -67,7 +67,7 @@ def env(self,a=None,initial=None): # environment function, kernel uses it to in def loss(self,s,a,next_s,r,d): # loss function, kernel uses it to calculate loss a=tf.expand_dims(a,axis=1) # expand the action vector to match the Q-value matrix shape next_q_value=self.target_critic.fp(next_s,self.target_actor.fp(next_s)) # get the Q-value for the next state and action from the target critic network - q_target=r+self.gamma*next_q_value*(1-tf.cast(d,'float32')) # calculate the target value using Bellman equation with discount factor gamma + q_target=tf.cast(r,'float32')+self.gamma*next_q_value*(1-tf.cast(d,'float32')) # calculate the target value using Bellman equation with discount factor gamma actor_loss=-tf.reduce_mean(self.critic.fp(s,self.actor.fp(s))) # calculate the actor loss as the negative mean of the Q-value for the current state and action from the critic network critic_loss=tf.reduce_mean((self.critic.fp(s,a)-q_target)**2) # calculate the critic loss as the mean squared error between Q-value and target value return [actor_loss,critic_loss] # return a list of actor loss and critic loss diff --git a/7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN.py b/7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN.py index b8ea6f1c..0f7d8d0b 100644 --- a/7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN.py +++ b/7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN.py @@ -36,7 +36,7 @@ def loss(self,s,a,next_s,r,d): # loss function, kernel uses it to calculate loss a=tf.expand_dims(a,axis=1) # expand the action vector to match the Q-value matrix shape q_value=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) # get the Q-value for the selected action next_q_value=tf.reduce_max(self.target_q_net.fp(next_s),axis=1) # get the maximum Q-value for the next state from the target network - target=r+0.98*next_q_value*(1-tf.cast(d,'float32')) # calculate the target value using Bellman equation with discount factor 0.98 + target=tf.cast(r,'float32')+0.98*next_q_value*(1-tf.cast(d,'float32')) # calculate the target value using Bellman equation with discount factor 0.98 return tf.reduce_mean((q_value-target)**2) # return the mean squared error between Q-value and target value diff --git a/7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN_pr.py b/7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN_pr.py index 1f901d5d..1a8d81f2 100644 --- a/7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN_pr.py +++ b/7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN_pr.py @@ -46,7 +46,7 @@ def loss(self,s,a,next_s,r,d): a=tf.expand_dims(a,axis=1) q_value=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) next_q_value=tf.reduce_max(self.target_q_net.fp(next_s),axis=1) - target=r+0.98*next_q_value*(1-tf.cast(d,'float32')) + target=tf.cast(r,'float32')+0.98*next_q_value*(1-tf.cast(d,'float32')) target=tf.expand_dims(target,axis=1) TD=target-q_value self.pr.update_TD(TD) diff --git a/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py b/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py index 092fc7c4..1efbfee0 100644 --- a/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py +++ b/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py @@ -53,7 +53,7 @@ def loss(self,s,a,next_s,r,d): action_prob_old=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) raito=action_prob/action_prob_old value=self.critic.fp(s) - value_tar=r+0.98*self.critic.fp(next_s)*(1-tf.cast(d,'float32')) + value_tar=tf.cast(r,'float32')+0.98*self.critic.fp(next_s)*(1-tf.cast(d,'float32')) TD=value_tar-value sur1=raito*TD sur2=tf.clip_by_value(raito,clip_value_min=1-self.clip_eps,clip_value_max=1+self.clip_eps)*TD diff --git a/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO_eps.py b/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO_eps.py index 947cea51..f17b625c 100644 --- a/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO_eps.py +++ b/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO_eps.py @@ -50,7 +50,7 @@ def loss(self,s,a,next_s,r,d): a=tf.expand_dims(a,axis=1) raito=tf.gather(self.actor.fp(s),a,axis=1,batch_dims=1)/tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) value=self.critic.fp(s) - value_tar=r+0.98*self.critic.fp(next_s)*(1-tf.cast(d,'float32')) + value_tar=tf.cast(r,'float32')+0.98*self.critic.fp(next_s)*(1-tf.cast(d,'float32')) TD=value_tar-value sur1=raito*TD sur2=tf.clip_by_value(raito,clip_value_min=1-self.clip_eps,clip_value_max=1+self.clip_eps)*TD From c98ad386f92a42d76b16eac68d44f6c83a245b4f Mon Sep 17 00:00:00 2001 From: NoteDance Date: Fri, 19 Jul 2024 19:17:24 +0800 Subject: [PATCH 308/337] Add files via upload --- 7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN.py | 2 +- 7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN_pr.py | 2 +- 7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py | 2 +- 7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO_eps.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN.py b/7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN.py index 0f7d8d0b..d0be2840 100644 --- a/7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN.py +++ b/7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN.py @@ -41,5 +41,5 @@ def loss(self,s,a,next_s,r,d): # loss function, kernel uses it to calculate loss def update_param(self): # update function, kernel uses it to update parameter - nn.assign_param(self.target_q_net.param, self.param.copy()) # copy the parameters from the Q-network to the target network + nn.assign_param(self.target_q_net.param, self.param) # copy the parameters from the Q-network to the target network return \ No newline at end of file diff --git a/7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN_pr.py b/7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN_pr.py index 1a8d81f2..e907903a 100644 --- a/7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN_pr.py +++ b/7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN_pr.py @@ -54,5 +54,5 @@ def loss(self,s,a,next_s,r,d): def update_param(self): - nn.assign_param(self.target_q_net.param, self.param.copy()) + nn.assign_param(self.target_q_net.param, self.param) return \ No newline at end of file diff --git a/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py b/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py index 1efbfee0..6f9cfb42 100644 --- a/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py +++ b/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py @@ -78,5 +78,5 @@ def opt(self,gradient): def update_param(self): - nn.assign_param(self.nn.param, self.actor.param.copy()) + nn.assign_param(self.nn.param, self.actor.param) return \ No newline at end of file diff --git a/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO_eps.py b/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO_eps.py index f17b625c..77ebe9c3 100644 --- a/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO_eps.py +++ b/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO_eps.py @@ -72,5 +72,5 @@ def opt(self,gradient): def update_param(self): - nn.assign_param(self.nn.param, self.actor.param.copy()) + nn.assign_param(self.nn.param, self.actor.param) return \ No newline at end of file From 8ed416b01ad9d68000f2c05f4268c0a66576f68f Mon Sep 17 00:00:00 2001 From: NoteDance Date: Fri, 19 Jul 2024 19:18:20 +0800 Subject: [PATCH 309/337] Add files via upload --- 7.0/neuralnetwork/RL/tensorflow/parallel/DQN.py | 2 +- 7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py | 2 +- 7.0/neuralnetwork/RL/tensorflow/parallel/PPO_eps.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/7.0/neuralnetwork/RL/tensorflow/parallel/DQN.py b/7.0/neuralnetwork/RL/tensorflow/parallel/DQN.py index 889e7b03..9186307d 100644 --- a/7.0/neuralnetwork/RL/tensorflow/parallel/DQN.py +++ b/7.0/neuralnetwork/RL/tensorflow/parallel/DQN.py @@ -42,7 +42,7 @@ def loss(self,s,a,next_s,r,d): # loss function, kernel uses it to calculate loss def update_param(self): # update function, kernel uses it to update parameter - nn.assign(self.target_q_net.param,self.param.copy()) # copy the parameters from the Q-network to the target network + nn.assign(self.target_q_net.param,self.param) # copy the parameters from the Q-network to the target network return diff --git a/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py b/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py index 29923b2b..61b81f79 100644 --- a/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py +++ b/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py @@ -77,5 +77,5 @@ def opt(self,gradient): def update_param(self): - nn.assign(self.nn.param,self.actor.param.copy()) + nn.assign(self.nn.param,self.actor.param) return \ No newline at end of file diff --git a/7.0/neuralnetwork/RL/tensorflow/parallel/PPO_eps.py b/7.0/neuralnetwork/RL/tensorflow/parallel/PPO_eps.py index 3161a91a..0fc75aa4 100644 --- a/7.0/neuralnetwork/RL/tensorflow/parallel/PPO_eps.py +++ b/7.0/neuralnetwork/RL/tensorflow/parallel/PPO_eps.py @@ -73,5 +73,5 @@ def opt(self,gradient): def update_param(self): - nn.assign(self.nn.param,self.actor.param.copy()) + nn.assign(self.nn.param,self.actor.param) return \ No newline at end of file From 09d16a070c759dff8e9d1c39cdb96395adf67bd5 Mon Sep 17 00:00:00 2001 From: NoteDance Date: Fri, 26 Jul 2024 18:12:24 +0800 Subject: [PATCH 310/337] Add files via upload --- .../RL/tensorflow/non_parallel/DDPG.py | 2 +- .../RL/tensorflow/non_parallel/PPO.py | 16 +--------------- .../RL/tensorflow/non_parallel/PPO_eps.py | 15 +-------------- 3 files changed, 3 insertions(+), 30 deletions(-) diff --git a/7.0/neuralnetwork/RL/tensorflow/non_parallel/DDPG.py b/7.0/neuralnetwork/RL/tensorflow/non_parallel/DDPG.py index 47c59e65..b728c4b3 100644 --- a/7.0/neuralnetwork/RL/tensorflow/non_parallel/DDPG.py +++ b/7.0/neuralnetwork/RL/tensorflow/non_parallel/DDPG.py @@ -70,7 +70,7 @@ def loss(self,s,a,next_s,r,d): # loss function, kernel uses it to calculate los q_target=tf.cast(r,'float32')+self.gamma*next_q_value*(1-tf.cast(d,'float32')) # calculate the target value using Bellman equation with discount factor gamma actor_loss=-tf.reduce_mean(self.critic.fp(s,self.actor.fp(s))) # calculate the actor loss as the negative mean of the Q-value for the current state and action from the critic network critic_loss=tf.reduce_mean((self.critic.fp(s,a)-q_target)**2) # calculate the critic loss as the mean squared error between Q-value and target value - return [actor_loss,critic_loss] # return a list of actor loss and critic loss + return actor_loss+critic_loss # return a list of actor loss and critic loss def update_param(self): # update function, kernel uses it to update parameter diff --git a/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py b/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py index 6f9cfb42..c0e9b704 100644 --- a/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py +++ b/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py @@ -60,23 +60,9 @@ def loss(self,s,a,next_s,r,d): clip_loss=-tf.math.minimum(sur1,sur2) entropy=action_prob*tf.math.log(action_prob+1e-8) clip_loss=clip_loss-self.alpha*entropy - return [tf.reduce_mean(clip_loss),tf.reduce_mean((TD)**2)] + return tf.reduce_mean(clip_loss)+tf.reduce_mean((TD)**2) - - def gradient(self,tape,loss): - actor_gradient=tape.gradient(loss[0],self.param[0]) - critic_gradient=tape.gradient(loss[1],self.param[1]) - return [actor_gradient,critic_gradient] - - - def opt(self,gradient): - self.opt.apply_gradients(zip(gradient[0],self.param[0])) - self.opt.apply_gradients(zip(gradient[1],self.param[1])) - self.actor_old.param=self.actor.param.copy() - return - - def update_param(self): nn.assign_param(self.nn.param, self.actor.param) return \ No newline at end of file diff --git a/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO_eps.py b/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO_eps.py index 77ebe9c3..1287521b 100644 --- a/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO_eps.py +++ b/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO_eps.py @@ -55,20 +55,7 @@ def loss(self,s,a,next_s,r,d): sur1=raito*TD sur2=tf.clip_by_value(raito,clip_value_min=1-self.clip_eps,clip_value_max=1+self.clip_eps)*TD clip_loss=-tf.math.minimum(sur1,sur2) - return [tf.reduce_mean(clip_loss),tf.reduce_mean((TD)**2)] - - - def gradient(self,tape,loss): - actor_gradient=tape.gradient(loss[0],self.param[0]) - critic_gradient=tape.gradient(loss[1],self.param[1]) - return [actor_gradient,critic_gradient] - - - def opt(self,gradient): - self.opt.apply_gradients(zip(gradient[0],self.param[0])) - self.opt.apply_gradients(zip(gradient[1],self.param[1])) - self.actor_old.param=self.actor.param.copy() - return + return tf.reduce_mean(clip_loss)+tf.reduce_mean((TD)**2) def update_param(self): From abf24fe0cd50f9bd9638fbb8641184cb0d590e65 Mon Sep 17 00:00:00 2001 From: NoteDance Date: Fri, 26 Jul 2024 18:13:35 +0800 Subject: [PATCH 311/337] Add files via upload --- 7.0/neuralnetwork/RL/tensorflow/parallel/DDPG.py | 2 +- 7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py | 15 +-------------- .../RL/tensorflow/parallel/PPO_eps.py | 15 +-------------- 3 files changed, 3 insertions(+), 29 deletions(-) diff --git a/7.0/neuralnetwork/RL/tensorflow/parallel/DDPG.py b/7.0/neuralnetwork/RL/tensorflow/parallel/DDPG.py index 35084078..b784eb5c 100644 --- a/7.0/neuralnetwork/RL/tensorflow/parallel/DDPG.py +++ b/7.0/neuralnetwork/RL/tensorflow/parallel/DDPG.py @@ -71,7 +71,7 @@ def loss(self,s,a,next_s,r,d): # loss function, kernel uses it to calculate los q_target=tf.cast(r,'float32')+self.gamma*next_q_value*(1-tf.cast(d,'float32')) # calculate the target value using Bellman equation with discount factor gamma actor_loss=-tf.reduce_mean(self.critic.fp(s,self.actor.fp(s))) # calculate the actor loss as the negative mean of the Q-value for the current state and action from the critic network critic_loss=tf.reduce_mean((self.critic.fp(s,a)-q_target)**2) # calculate the critic loss as the mean squared error between Q-value and target value - return [actor_loss,critic_loss] # return a list of actor loss and critic loss + return actor_loss+critic_loss # return a list of actor loss and critic loss def update_param(self): # update function, kernel uses it to update parameter diff --git a/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py b/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py index 61b81f79..88763150 100644 --- a/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py +++ b/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py @@ -60,20 +60,7 @@ def loss(self,s,a,next_s,r,d): clip_loss=-tf.math.minimum(sur1,sur2) entropy=action_prob*tf.math.log(action_prob+1e-8) clip_loss=clip_loss-self.alpha*entropy - return [tf.reduce_mean(clip_loss),tf.reduce_mean((TD)**2)] - - - def gradient(self,tape,loss): - actor_gradient=tape.gradient(loss[0],self.param[0]) - critic_gradient=tape.gradient(loss[1],self.param[1]) - return [actor_gradient,critic_gradient] - - - def opt(self,gradient): - self.opt.opt(gradient[0],self.param[0]) - self.opt.opt(gradient[1],self.param[1]) - self.actor_old.param=self.actor.param.copy() - return + return tf.reduce_mean(clip_loss)+tf.reduce_mean((TD)**2) def update_param(self): diff --git a/7.0/neuralnetwork/RL/tensorflow/parallel/PPO_eps.py b/7.0/neuralnetwork/RL/tensorflow/parallel/PPO_eps.py index 0fc75aa4..008baf04 100644 --- a/7.0/neuralnetwork/RL/tensorflow/parallel/PPO_eps.py +++ b/7.0/neuralnetwork/RL/tensorflow/parallel/PPO_eps.py @@ -56,20 +56,7 @@ def loss(self,s,a,next_s,r,d): sur1=raito*TD sur2=tf.clip_by_value(raito,clip_value_min=1-self.clip_eps,clip_value_max=1+self.clip_eps)*TD clip_loss=-tf.math.minimum(sur1,sur2) - return [tf.reduce_mean(clip_loss),tf.reduce_mean((TD)**2)] - - - def gradient(self,tape,loss): - actor_gradient=tape.gradient(loss[0],self.param[0]) - critic_gradient=tape.gradient(loss[1],self.param[1]) - return [actor_gradient,critic_gradient] - - - def opt(self,gradient): - self.opt.opt(gradient[0],self.param[0]) - self.opt.opt(gradient[1],self.param[1]) - self.actor_old.param=self.actor.param.copy() - return + return tf.reduce_mean(clip_loss)+tf.reduce_mean((TD)**2) def update_param(self): From 53539a64f5cbb46deb2cd70eaccfed775b911c66 Mon Sep 17 00:00:00 2001 From: NoteDance Date: Wed, 31 Jul 2024 19:20:27 +0800 Subject: [PATCH 312/337] Update DDPG.py --- 7.0/neuralnetwork/RL/pytorch/non_parallel/DDPG.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/7.0/neuralnetwork/RL/pytorch/non_parallel/DDPG.py b/7.0/neuralnetwork/RL/pytorch/non_parallel/DDPG.py index 3a6cdb28..cf04915d 100644 --- a/7.0/neuralnetwork/RL/pytorch/non_parallel/DDPG.py +++ b/7.0/neuralnetwork/RL/pytorch/non_parallel/DDPG.py @@ -1,6 +1,5 @@ import torch import torch.nn.functional as F -import numpy as np import gym @@ -53,10 +52,6 @@ def __init__(self,hidden_dim,sigma,gamma,tau,actor_lr,critic_lr): self.critic_opt=torch.optim.Adam(self.critic.parameters(),lr=critic_lr) - def noise(self): - return np.random.normal(scale=self.sigma) - - def env(self,a=None,initial=None): if initial==True: state=self.genv.reset(seed=0) From ac6650b3de8c76c0aee51408500e204689f392ae Mon Sep 17 00:00:00 2001 From: NoteDance Date: Wed, 31 Jul 2024 19:20:45 +0800 Subject: [PATCH 313/337] Update DDPG.py --- 7.0/neuralnetwork/RL/tensorflow/non_parallel/DDPG.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/7.0/neuralnetwork/RL/tensorflow/non_parallel/DDPG.py b/7.0/neuralnetwork/RL/tensorflow/non_parallel/DDPG.py index b728c4b3..2f4aa1e2 100644 --- a/7.0/neuralnetwork/RL/tensorflow/non_parallel/DDPG.py +++ b/7.0/neuralnetwork/RL/tensorflow/non_parallel/DDPG.py @@ -1,6 +1,5 @@ import tensorflow as tf # import TensorFlow library from Note import nn -import numpy as np # import NumPy library import gym # import OpenAI Gym library @@ -51,10 +50,6 @@ def __init__(self,hidden_dim,sigma,gamma,tau,actor_lr,critic_lr): # initialize t self.opt=tf.keras.optimizers.Adam() # optimizer, kernel uses it to optimize. Here we use Adam optimizer - def noise(self): # noise function, kernel uses it to generate exploration noise - return np.random.normal(scale=self.sigma) # return a random sample from a normal distribution with zero mean and sigma scale - - def env(self,a=None,initial=None): # environment function, kernel uses it to interact with the environment if initial==True: # if initial is True, reset the environment and return the initial state state=self.genv.reset(seed=0) @@ -78,4 +73,4 @@ def update_param(self): # update function, kernel uses it to update parameter target_param.assign(target_param*(1.0-self.tau)+param*self.tau) # update the target parameter using soft update with factor tau for target_param,param in zip(self.target_critic.param,self.critic.param): # for each pair of parameters in target critic network and critic network target_param.assign(target_param*(1.0-self.tau)+param*self.tau) # update the target parameter using soft update with factor tau - return \ No newline at end of file + return From 65bc63e9320483816d9eb2ee4c2bdbf008f4bc2a Mon Sep 17 00:00:00 2001 From: NoteDance Date: Wed, 31 Jul 2024 19:20:59 +0800 Subject: [PATCH 314/337] Update DDPG.py --- 7.0/neuralnetwork/RL/tensorflow/parallel/DDPG.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/7.0/neuralnetwork/RL/tensorflow/parallel/DDPG.py b/7.0/neuralnetwork/RL/tensorflow/parallel/DDPG.py index b784eb5c..4235f50e 100644 --- a/7.0/neuralnetwork/RL/tensorflow/parallel/DDPG.py +++ b/7.0/neuralnetwork/RL/tensorflow/parallel/DDPG.py @@ -1,6 +1,5 @@ import tensorflow as tf # import TensorFlow library from Note import nn -import numpy as np # import NumPy library import gym # import OpenAI Gym library import Note.nn.parallel.optimizer as o # import Note's optimizer module @@ -52,10 +51,6 @@ def __init__(self,hidden_dim,sigma,gamma,tau,actor_lr,critic_lr): # initialize t self.optimizer=o.SGD(param=self.param) # optimizer, kernel uses it to optimize. Here we use a custom SGD optimizer - def noise(self): # noise function, kernel uses it to generate exploration noise - return np.random.normal(scale=self.sigma) # return a random sample from a normal distribution with zero mean and sigma scale - - def env(self,a=None,p=None,initial=None): # environment function, kernel uses it to interact with the environment if initial==True: # if initial is True, reset the environment with index p and return the initial state state=self.genv[p].reset(seed=0) @@ -79,4 +74,4 @@ def update_param(self): # update function, kernel uses it to update parameter target_param.assign(target_param*(1.0-self.tau)+param*self.tau) # update the target parameter using soft update with factor tau for target_param,param in zip(self.target_critic.param,self.critic.param): # for each pair of parameters in target critic network and critic network target_param.assign(target_param*(1.0-self.tau)+param*self.tau) # update the target parameter using soft update with factor tau - return \ No newline at end of file + return From cb35d777f85b83accadfaaaed370435a5ded39ce Mon Sep 17 00:00:00 2001 From: NoteDance Date: Tue, 6 Aug 2024 09:47:52 +0800 Subject: [PATCH 315/337] Delete 7.0/neuralnetwork directory --- .../DL/pytorch/non_parallel/nn.py | 53 ---------- 7.0/neuralnetwork/DL/pytorch/parallel/nn.py | 41 -------- .../DL/pytorch/parallel/nn_device.py | 45 --------- .../DL/tensorflow/non_parallel/cnn.py | 27 ------ .../DL/tensorflow/non_parallel/layer/cnn.py | 48 ---------- .../DL/tensorflow/non_parallel/layer/nn.py | 38 -------- .../DL/tensorflow/non_parallel/nn.py | 23 ----- .../DL/tensorflow/non_parallel/nn_acc.py | 28 ------ .../DL/tensorflow/non_parallel/nn_ol.py | 37 ------- .../DL/tensorflow/parallel/cnn.py | 51 ---------- .../DL/tensorflow/parallel/nn.py | 41 -------- .../DL/tensorflow/parallel/nn_acc.py | 46 --------- .../DL/tensorflow/parallel/nn_attenuate.py | 56 ----------- .../DL/tensorflow/parallel/nn_clipping.py | 49 ---------- .../DL/tensorflow/parallel/nn_device.py | 50 ---------- .../RL/pytorch/non_parallel/DDPG.py | 96 ------------------- .../RL/pytorch/non_parallel/DQN.py | 64 ------------- .../RL/pytorch/non_parallel/DQN_pr.py | 76 --------------- .../RL/pytorch/non_parallel/DoubleDQN.py | 65 ------------- .../RL/pytorch/non_parallel/DuelingDQN.py | 67 ------------- 7.0/neuralnetwork/RL/pytorch/parallel/DQN.py | 65 ------------- .../RL/tensorflow/non_parallel/DDPG.py | 76 --------------- .../RL/tensorflow/non_parallel/DQN.py | 45 --------- .../RL/tensorflow/non_parallel/DQN_pr.py | 58 ----------- .../RL/tensorflow/non_parallel/PPO.py | 68 ------------- .../RL/tensorflow/non_parallel/PPO_eps.py | 63 ------------ .../RL/tensorflow/parallel/DDPG.py | 77 --------------- .../RL/tensorflow/parallel/DQN.py | 51 ---------- .../RL/tensorflow/parallel/PPO.py | 68 ------------- .../RL/tensorflow/parallel/PPO_eps.py | 64 ------------- 30 files changed, 1636 deletions(-) delete mode 100644 7.0/neuralnetwork/DL/pytorch/non_parallel/nn.py delete mode 100644 7.0/neuralnetwork/DL/pytorch/parallel/nn.py delete mode 100644 7.0/neuralnetwork/DL/pytorch/parallel/nn_device.py delete mode 100644 7.0/neuralnetwork/DL/tensorflow/non_parallel/cnn.py delete mode 100644 7.0/neuralnetwork/DL/tensorflow/non_parallel/layer/cnn.py delete mode 100644 7.0/neuralnetwork/DL/tensorflow/non_parallel/layer/nn.py delete mode 100644 7.0/neuralnetwork/DL/tensorflow/non_parallel/nn.py delete mode 100644 7.0/neuralnetwork/DL/tensorflow/non_parallel/nn_acc.py delete mode 100644 7.0/neuralnetwork/DL/tensorflow/non_parallel/nn_ol.py delete mode 100644 7.0/neuralnetwork/DL/tensorflow/parallel/cnn.py delete mode 100644 7.0/neuralnetwork/DL/tensorflow/parallel/nn.py delete mode 100644 7.0/neuralnetwork/DL/tensorflow/parallel/nn_acc.py delete mode 100644 7.0/neuralnetwork/DL/tensorflow/parallel/nn_attenuate.py delete mode 100644 7.0/neuralnetwork/DL/tensorflow/parallel/nn_clipping.py delete mode 100644 7.0/neuralnetwork/DL/tensorflow/parallel/nn_device.py delete mode 100644 7.0/neuralnetwork/RL/pytorch/non_parallel/DDPG.py delete mode 100644 7.0/neuralnetwork/RL/pytorch/non_parallel/DQN.py delete mode 100644 7.0/neuralnetwork/RL/pytorch/non_parallel/DQN_pr.py delete mode 100644 7.0/neuralnetwork/RL/pytorch/non_parallel/DoubleDQN.py delete mode 100644 7.0/neuralnetwork/RL/pytorch/non_parallel/DuelingDQN.py delete mode 100644 7.0/neuralnetwork/RL/pytorch/parallel/DQN.py delete mode 100644 7.0/neuralnetwork/RL/tensorflow/non_parallel/DDPG.py delete mode 100644 7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN.py delete mode 100644 7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN_pr.py delete mode 100644 7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py delete mode 100644 7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO_eps.py delete mode 100644 7.0/neuralnetwork/RL/tensorflow/parallel/DDPG.py delete mode 100644 7.0/neuralnetwork/RL/tensorflow/parallel/DQN.py delete mode 100644 7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py delete mode 100644 7.0/neuralnetwork/RL/tensorflow/parallel/PPO_eps.py diff --git a/7.0/neuralnetwork/DL/pytorch/non_parallel/nn.py b/7.0/neuralnetwork/DL/pytorch/non_parallel/nn.py deleted file mode 100644 index 1faec0fc..00000000 --- a/7.0/neuralnetwork/DL/pytorch/non_parallel/nn.py +++ /dev/null @@ -1,53 +0,0 @@ -import torch -from torch import nn - - -class NeuralNetwork(nn.Module): - def __init__(self): - super().__init__() - self.flatten=nn.Flatten() - self.linear_relu_stack=nn.Sequential( - nn.Linear(28*28, 512), - nn.ReLU(), - nn.Linear(512, 512), - nn.ReLU(), - nn.Linear(512, 10) - ) - - - def forward(self,x): - x=self.flatten(x) - logits=self.linear_relu_stack(x) - return logits - - -class neuralnetwork: - def __init__(self): - if torch.cuda.is_available(): - self.device=torch.device('cuda') - else: - self.device=torch.device('cpu') - self.model=NeuralNetwork().to(self.device) - self.loss_fn=nn.CrossEntropyLoss() - self.opt=torch.optim.SGD(self.model.parameters(),lr=1e-3) #optimizer,kernel uses it to optimize. - - - def fp(self,x): #forward propagation function,kernel uses it for forward propagation. - pred=self.model(x.to(self.device)) - return pred - - - def loss(self,output,labels): #loss functino,kernel uses it to calculate loss. - loss=self.loss_fn(output,labels.to(self.device)) - return loss - - - def backward(self,loss): #backward function,kernel uses it for backpropagation. - self.optim.zero_grad() - loss.backward() - return - - - def opt(self): #opt function,kernel uses it to optimize. - self.optim.step() - return \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/pytorch/parallel/nn.py b/7.0/neuralnetwork/DL/pytorch/parallel/nn.py deleted file mode 100644 index d9b05567..00000000 --- a/7.0/neuralnetwork/DL/pytorch/parallel/nn.py +++ /dev/null @@ -1,41 +0,0 @@ -import torch # import the PyTorch library -from torch import nn # import the neural network module from PyTorch - -class NeuralNetwork(nn.Module): # define a class for the neural network model - def __init__(self): # define the constructor method - super().__init__() # call the parent class constructor - self.flatten = nn.Flatten() # define a layer to flatten the input - self.linear_relu_stack=nn.Sequential( # define a stack of linear and relu layers - nn.Linear(28*28,512), # a linear layer with 28*28 input features and 512 output features - nn.ReLU(), # a relu activation function - nn.Linear(512,512), # another linear layer with 512 input and output features - nn.ReLU(), # another relu activation function - nn.Linear(512,10) # a final linear layer with 512 input features and 10 output features - ) - - - def forward(self,x): # define the forward method - x = self.flatten(x) # flatten the input x - logits=self.linear_relu_stack(x) # pass x through the linear relu stack and get the logits - return logits # return the logits - - -class neuralnetwork: # define another class for the neural network object - def __init__(self): # define the constructor method - if torch.cuda.is_available(): # check if cuda is available - self.device=torch.device('cuda') # set the device to cuda - else: - self.device=torch.device('cpu') # otherwise set the device to cpu - self.model=NeuralNetwork().to(self.device) # create an instance of the NeuralNetwork class and move it to the device - self.loss_fn=nn.CrossEntropyLoss() # define the loss function as cross entropy loss - self.opt=[torch.optim.SGD(self.model.parameters(),lr=1e-3) for _ in range(7)] # define a list of three stochastic gradient descent optimizers with learning rate 1e-3 - - - def fp(self,x): # define a method for forward propagation - pred=self.model(x.to(self.device)) # get the predictions from the model by passing x to the device - return pred # return the predictions - - - def loss(self,output,labels): # define a method for calculating the loss - loss=self.loss_fn(output,labels.to(self.device)) # compute the loss by passing output and labels to the device - return loss # return the loss \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/pytorch/parallel/nn_device.py b/7.0/neuralnetwork/DL/pytorch/parallel/nn_device.py deleted file mode 100644 index ad4b8069..00000000 --- a/7.0/neuralnetwork/DL/pytorch/parallel/nn_device.py +++ /dev/null @@ -1,45 +0,0 @@ -import torch # import the PyTorch library -from torch import nn # import the neural network module from PyTorch -from Note.nn.parallel.assign_device_pytorch import assign_device # import a custom function to assign device for PyTorch tensors - - -class NeuralNetwork(nn.Module): # define a class for the neural network model - def __init__(self): # define the constructor method - super().__init__() # call the parent class constructor - self.flatten = nn.Flatten() # define a layer to flatten the input tensor - self.linear_relu_stack=nn.Sequential( # define a stack of linear and relu layers - nn.Linear(28*28,512), # define a linear layer with input size 28*28 and output size 512 - nn.ReLU(), # define a relu activation function - nn.Linear(512,512), # define another linear layer with input and output size 512 - nn.ReLU(), # define another relu activation function - nn.Linear(512,10) # define the final linear layer with output size 10 - ) - - - def forward(self,x): # define the forward method for the model - x = self.flatten(x) # flatten the input tensor - logits=self.linear_relu_stack(x) # pass the flattened tensor through the linear relu stack - return logits # return the logits as the output - - -class neuralnetwork: # define another class for the neural network object - def __init__(self): # define the constructor method - if torch.cuda.is_available(): # check if cuda device is available - self.device=torch.device('cuda') # set the device to cuda - else: # otherwise - self.device=torch.device('cpu') # set the device to cpu - self.model=NeuralNetwork().to(self.device) # create an instance of the neural network model and move it to the device - self.loss_fn=nn.CrossEntropyLoss() # define the loss function as cross entropy loss - self.opt=[torch.optim.SGD(self.model.parameters(),lr=1e-3) for _ in range(7)] # define a list of optimizers as stochastic gradient descent with learning rate 1e-3 - - - def fp(self,x,p): # define a method for forward propagation - pred=self.model(x.to(assign_device(p,'GPU'))) # assign the device according to the process index p - # get the predictions from the model by passing the input tensor to the device and then to the model - return pred # return the predictions - - - def loss(self,output,labels,p): # define a method for calculating loss - loss=self.loss_fn(output,labels.to(assign_device(p,'GPU'))) # assign the device according to the process index p - # calculate the loss by passing the output and labels tensors to the device and then to the loss function - return loss # return the loss \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/non_parallel/cnn.py b/7.0/neuralnetwork/DL/tensorflow/non_parallel/cnn.py deleted file mode 100644 index bd7ad1bf..00000000 --- a/7.0/neuralnetwork/DL/tensorflow/non_parallel/cnn.py +++ /dev/null @@ -1,27 +0,0 @@ -import tensorflow as tf -from tensorflow.keras import layers,models - - -class cnn: - def __init__(self): - self.model=models.Sequential() - self.model.add(layers.Conv2D(32,(3,3),activation='relu',input_shape=(32,32,3))) - self.model.add(layers.MaxPooling2D((2,2))) - self.model.add(layers.Conv2D(64,(3,3),activation='relu')) - self.model.add(layers.MaxPooling2D((2,2))) - self.model.add(layers.Conv2D(64,(3,3),activation='relu')) - self.model.add(layers.Flatten()) - self.model.add(layers.Dense(64,activation='relu')) - self.model.add(layers.Dense(10)) - self.param=self.model.weights - self.opt=tf.keras.optimizers.Adam() - - - def fp(self,data): - output=self.model(data) - return output - - - def loss(self,output,labels): - loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - return loss(labels,output) \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/non_parallel/layer/cnn.py b/7.0/neuralnetwork/DL/tensorflow/non_parallel/layer/cnn.py deleted file mode 100644 index cf396557..00000000 --- a/7.0/neuralnetwork/DL/tensorflow/non_parallel/layer/cnn.py +++ /dev/null @@ -1,48 +0,0 @@ -import tensorflow as tf -from Note.nn.layer.conv2d import conv2d -from Note.nn.layer.dense import dense -from Note.nn.layer.flatten import flatten -from Note.nn.Model import Model - -""" -This is an example of using the Note layer module. -""" - -# Define a convolutional neural network class -class cnn: - def __init__(self): - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.opt=tf.keras.optimizers.Adam() - - - def build(self): - # Create three convolutional layers with relu activations - self.conv1=conv2d(32,[3,3],strides=(1,1),padding='SAME',activation='relu') - self.conv2=conv2d(64,[3,3],strides=(2,2),padding='SAME',activation='relu') - self.conv3=conv2d(64,[3,3],strides=(1,1),padding='SAME',activation='relu') - self.flatten=flatten() - # Create two dense layers with relu and linear activations - self.dense1=dense(64,activation='relu') - self.dense2=dense(10) - # Store the parameters of the layers in a list - self.param=Model.param - return - - - def fp(self,data): - # Perform forward propagation on the input data - x=self.conv1(data) # First convolutional layer - x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # First max pooling layer - x=self.conv2(x) # Second convolutional layer - x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # Second max pooling layer - x=self.conv3(x) # Third convolutional layer - x=self.flatten(x) # Flatten the output to a vector - x=self.dense1(x) # First dense layer with relu activation - output=self.dense2.output(x) # Output layer with linear activation - return output - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/non_parallel/layer/nn.py b/7.0/neuralnetwork/DL/tensorflow/non_parallel/layer/nn.py deleted file mode 100644 index 7dc63176..00000000 --- a/7.0/neuralnetwork/DL/tensorflow/non_parallel/layer/nn.py +++ /dev/null @@ -1,38 +0,0 @@ -import tensorflow as tf -from Note.nn.layer.dense import dense -from Note.nn.layer.flatten import flatten -from Note.nn.Model import Model - -""" -This is an example of using the Note layer module. -""" - -# Define a neural network class -class nn: - def __init__(self): - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.opt=tf.keras.optimizers.Adam() #optimizer,kernel uses it to optimize. - - - def build(self): - # Create two dense layers with relu and linear activations - self.layer1=dense(128,activation='relu') - self.layer2=dense(10) - self.flatten=flatten() - # Store the parameters of the layers in a list - self.param=Model.param - return - - - def fp(self,data): - # Perform forward propagation on the input data - data=self.flatten(data) - output1=self.layer1(data) - output2=self.layer2(output1) - return output2 - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/non_parallel/nn.py b/7.0/neuralnetwork/DL/tensorflow/non_parallel/nn.py deleted file mode 100644 index b4f03a92..00000000 --- a/7.0/neuralnetwork/DL/tensorflow/non_parallel/nn.py +++ /dev/null @@ -1,23 +0,0 @@ -import tensorflow as tf - - -class nn: - def __init__(self): - self.model=tf.keras.models.Sequential([ - tf.keras.layers.Flatten(input_shape=(28, 28)), - tf.keras.layers.Dense(128,activation='relu'), - tf.keras.layers.Dropout(0.2), - tf.keras.layers.Dense(10) - ]) - self.param=self.model.weights #parameter list,kernel uses it list for backpropagation. - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.opt=tf.keras.optimizers.Adam() #optimizer,kernel uses it to optimize. - - - def fp(self,data): #forward propagation function,kernel uses it for forward propagation. - output=self.model(data) - return output - - - def loss(self,output,labels): #loss functino,kernel uses it to calculate loss. - return self.loss_object(labels,output) \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/non_parallel/nn_acc.py b/7.0/neuralnetwork/DL/tensorflow/non_parallel/nn_acc.py deleted file mode 100644 index 0a726603..00000000 --- a/7.0/neuralnetwork/DL/tensorflow/non_parallel/nn_acc.py +++ /dev/null @@ -1,28 +0,0 @@ -import tensorflow as tf - -#An example with accuracy function. -class nn: - def __init__(self): - self.model=tf.keras.models.Sequential([ - tf.keras.layers.Flatten(input_shape=(28, 28)), - tf.keras.layers.Dense(128,activation='relu'), - tf.keras.layers.Dropout(0.2), - tf.keras.layers.Dense(10) - ]) - self.param=self.model.weights #parameter list,kernel uses it list for backpropagation. - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - self.train_accuracy=tf.keras.metrics.SparseCategoricalAccuracy(name='train_accuracy') - self.opt=tf.keras.optimizers.Adam() #optimizer,kernel uses it to optimize. - - - def fp(self,data): #forward propagation function,kernel uses it for forward propagation. - output=self.model(data) - return output - - - def loss(self,output,labels): #loss functino,kernel uses it to calculate loss. - return self.loss_object(labels,output) - - - def accuracy(self,output,labels): #accuracy function,kernel uses it to calculate accuracy. - return self.train_accuracy(labels,output) \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/non_parallel/nn_ol.py b/7.0/neuralnetwork/DL/tensorflow/non_parallel/nn_ol.py deleted file mode 100644 index 9dfa608d..00000000 --- a/7.0/neuralnetwork/DL/tensorflow/non_parallel/nn_ol.py +++ /dev/null @@ -1,37 +0,0 @@ -import tensorflow as tf # import TensorFlow library -import numpy as np # import NumPy library - -# This is an example of online training. -class nn: # define a class for the neural network - def __init__(self,data,labels): # initialize the network with data and labels - self.train_data=data # store the training data - self.train_labels=labels # store the training labels - self.model=tf.keras.models.Sequential([ # create a sequential model with four layers - tf.keras.layers.Flatten(input_shape=(28, 28)), # flatten the input to a vector of 28*28 elements - tf.keras.layers.Dense(128,activation='relu'), # add a dense layer with 128 units and ReLU activation - tf.keras.layers.Dropout(0.2), # add a dropout layer with 0.2 dropout rate to prevent overfitting - tf.keras.layers.Dense(10) # add a dense layer with 10 units for the output logits - ]) - self.param=self.model.weights # parameter list, kernel uses it list for backpropagation - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) # loss object, kernel uses it to calculate loss. Here we use sparse categorical crossentropy with logits as output - self.opt=tf.keras.optimizers.Adam() # optimizer, kernel uses it to optimize. Here we use Adam optimizer - self.train_loss_list=[] # a list to store the training loss values - self.counter=0 # counter to keep track of the number of online updates - self.max_length=1000 # maximum length of train_loss_list - - - def online(self): # This is simulative online function, kernel uses it to get online data and labels - index=np.random.choice(60000,size=[32]) # randomly sample 32 indices from 0 to 59999 - if self.counter==10000: # if the counter reaches 10000, stop the online training - return 'stop' - else: # otherwise, return the data and labels corresponding to the sampled indices - return [self.train_data[index],self.train_labels[index]] - - - def fp(self,data): # forward propagation function, kernel uses it for forward propagation - output=self.model(data) # pass the data through the model and get the output logits - return output - - - def loss(self,output,labels): # loss function, kernel uses it to calculate loss - return self.loss_object(labels,output) # return the sparse categorical crossentropy loss between labels and output \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/cnn.py b/7.0/neuralnetwork/DL/tensorflow/parallel/cnn.py deleted file mode 100644 index 6e084ee8..00000000 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/cnn.py +++ /dev/null @@ -1,51 +0,0 @@ -import tensorflow as tf -from Note.nn.layer.conv2d import conv2d -from Note.nn.layer.dense import dense -from Note.nn.layer.flatten import flatten -from Note.nn.parallel.optimizer import Adam -from Note.nn.Model import Model - -# Define a convolutional neural network class -class cnn: - def __init__(self): - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - - - def build(self): - # Create three convolutional layers with relu activations - self.conv1=conv2d(32,[3,3],3,strides=(1,1),padding='SAME',activation='relu') - self.conv2=conv2d(64,[3,3],32,strides=(2,2),padding='SAME',activation='relu') - self.conv3=conv2d(64,[3,3],64,strides=(1,1),padding='SAME',activation='relu') - self.flatten=flatten() - # Create two dense layers with relu and linear activations - self.dense1=dense(64,64,activation='relu') - self.dense2=dense(10,64) - self.optimizer=Adam() - # Store the parameters of the layers in a list - self.param=Model.param - return - - - def fp(self,data): - # Perform forward propagation on the input data - x=self.conv1(data) # First convolutional layer - x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # First max pooling layer - x=self.conv2(x) # Second convolutional layer - x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # Second max pooling layer - x=self.conv3(x) # Third convolutional layer - x=self.flatten(x) # Flatten the output to a vector - x=self.dense1(x) # First dense layer with relu activation - output=self.dense2.output(x) # Output layer with linear activation - return output - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) - - - def opt(self,gradient): - # Perform optimization on the parameters using the gradient and the batch count - param=self.optimizer(gradient,self.param,self.bc[0]) - return param \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn.py deleted file mode 100644 index db171189..00000000 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/nn.py +++ /dev/null @@ -1,41 +0,0 @@ -import tensorflow as tf -from Note.nn.layer.dense import dense -from Note.nn.parallel.optimizer import SGD -from Note.nn.layer.flatten import flatten -from Note.nn.Model import Model - -# Define a neural network class -class nn: - def __init__(self): - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - - - def build(self): - # Create two dense layers with relu and linear activations - self.layer1=dense(128,784,activation='relu') - self.layer2=dense(10,128) - self.flatten=flatten() - self.optimizer=SGD() - # Store the parameters of the layers in a list - self.param=Model.param - return - - - def fp(self,data): - # Perform forward propagation on the input data - data=self.flatten(data) - output1=self.layer1(data) - output2=self.layer2(output1) - return output2 - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) - - - def opt(self,gradient): - # Perform optimization on the parameters using the gradient - param=self.optimizer(gradient,self.param) - return param \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_acc.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_acc.py deleted file mode 100644 index cd3c8026..00000000 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_acc.py +++ /dev/null @@ -1,46 +0,0 @@ -import tensorflow as tf -from Note.nn.layer.dense import dense -from Note.nn.parallel.optimizer import SGD -from Note.nn.layer.flatten import flatten -from Note.nn.accuracy import sparse_categorical_accuracy -from Note.nn.Model import Model - -# Define a neural network class -class nn: - def __init__(self): - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - - - def build(self): - # Create two dense layers with relu and linear activations - self.layer1=dense(128,784,activation='relu') - self.layer2=dense(10,128) - self.flatten=flatten() - self.optimizer=SGD() - # Store the parameters of the layers in a list - self.param=Model.param - return - - - def fp(self,data): - # Perform forward propagation on the input data - data=self.flatten(data) - output1=self.layer1(data) - output2=self.layer2(output1) - return output2 - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) - - - def accuracy(self,output,labels): #accuracy function,kernel uses it to calculate accuracy. - return sparse_categorical_accuracy(labels,output) - - - def opt(self,gradient): - # Perform optimization on the parameters using the gradient - param=self.optimizer(gradient,self.param) - return param \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_attenuate.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_attenuate.py deleted file mode 100644 index 41f77f62..00000000 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_attenuate.py +++ /dev/null @@ -1,56 +0,0 @@ -import tensorflow as tf -from tensorflow.python.util import nest -from Note.nn.layer.dense import dense -from Note.nn.parallel.optimizer import SGD -from Note.nn.layer.flatten import flatten -from Note.nn.accuracy import sparse_categorical_accuracy -from Note.nn.Model import Model - -# Define a neural network class with gradient attenuation -class nn: - def __init__(self): - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - - - def build(self): - # Create two dense layers with relu and linear activations - self.layer1=dense(128,784,activation='relu') - self.layer2=dense(10,128) - self.flatten=flatten() - self.optimizer=SGD() - # Store the parameters of the layers in a list - self.param=Model.param - return - - - def fp(self,data): - # Perform forward propagation on the input data - data=self.flatten(data) # Flatten the data to a one-dimensional vector - output1=self.layer1(data) # Apply the first layer to the data and get the output - output2=self.layer2(output1) # Apply the second layer to the output of the first layer and get the final output - return output2 - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) # Use the loss function to calculate the loss - - - def accuracy(self,output,labels): #accuracy function,kernel uses it to calculate accuracy. - return sparse_categorical_accuracy(labels,output) - - - def attenuate(self,gradient,p): #gradient attenuation function,kernel uses it to calculate attenuation coefficient. - # Apply an exponential decay to the gradient based on the optimization counter - ac=0.9**self.opt_counter[0][p] #ac:attenuation coefficient - gradient_flat=nest.flatten(gradient) # Flatten the gradient to a one-dimensional vector - for i in range(len(gradient_flat)): #self.opt_counter:optimization counter - gradient_flat[i]=tf.cast(ac,gradient_flat[i].dtype)*gradient_flat[i] #p:process number - return gradient - - - def opt(self,gradient): - # Perform optimization on the parameters using the gradient - param=self.optimizer(gradient,self.param) # Use the optimizer to update the parameters - return param \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_clipping.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_clipping.py deleted file mode 100644 index 4929a49b..00000000 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_clipping.py +++ /dev/null @@ -1,49 +0,0 @@ -import tensorflow as tf -from Note.nn.layer.dense import dense -from Note.nn.parallel.optimizer import SGD -from Note.nn.layer.flatten import flatten -from Note.nn.gradient_clipping import gradient_clipping -from Note.nn.Model import Model - -# Define a neural network class -class nn: - def __init__(self): - # Initialize the loss function and the optimizer - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - - - def build(self): - # Create two dense layers with relu and linear activations - self.layer1=dense(128,784,activation='relu') - self.layer2=dense(10,128) - self.flatten=flatten() - self.optimizer=SGD() - # Store the parameters of the layers in a list - self.param=Model.param - return - - - def fp(self,data): - # Perform forward propagation on the input data - data=self.flatten(data) - output1=self.layer1(data) - output2=self.layer2(output1) - return output2 - - - def loss(self,output,labels): - # Compute the loss between the output and the labels - return self.loss_object(labels,output) - - - def gradient(self,tape,loss): - # Compute the gradients of the loss with respect to the parameters - grads=tape.gradient(loss,self.param) - # Clip the gradients by value to avoid exploding gradients - return gradient_clipping(grads,'value',1.0) - - - def opt(self,gradient): - # Perform optimization on the parameters using the gradient - param=self.optimizer(gradient,self.param) - return param \ No newline at end of file diff --git a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_device.py b/7.0/neuralnetwork/DL/tensorflow/parallel/nn_device.py deleted file mode 100644 index 7935af2e..00000000 --- a/7.0/neuralnetwork/DL/tensorflow/parallel/nn_device.py +++ /dev/null @@ -1,50 +0,0 @@ -import tensorflow as tf # import TensorFlow library -from Note.nn.layer.dense import dense # import dense layer class -from Note.nn.layer.flatten import flatten # import Note's flatten layer function -from Note.nn.parallel.optimizer import SGD # import Note's momentum optimizer module -from Note.nn.parallel.assign_device import assign_device # import the function to assign device according to the process index and the device type -from Note.nn.Model import Model - - -class nn: # A neural network class example, allocate device for multiple threads - def __init__(self): # initialize the network - self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) # loss object, kernel uses it to calculate loss. Here we use sparse categorical crossentropy with logits as output - - - def build(self): # build function, kernel uses it to create the network layers - # Create two dense layers with relu and linear activations - self.layer1=dense(128,784,activation='relu') # the first layer with 784 input units and 128 output units and ReLU activation - self.layer2=dense(10,128) # the second layer with 128 input units and 10 output units and linear activation - self.flatten=flatten() - self.optimizer=SGD() - # Store the parameters of the layers in a list - self.param=Model.param # parameter list of both layers, kernel uses it list for backpropagation - return - - - def fp(self,data,p): # forward propagation function, kernel uses it for forward propagation - with tf.device(assign_device(p,'GPU')): # assign the device according to the process index p - data=self.flatten(data) # flatten the data to a vector of 784 elements - output1=self.layer1(data) # pass the data through the first layer and get the output - output2=self.layer2(output1) # pass the output of the first layer through the second layer and get the final output logits - return output2 - - - def loss(self,output,labels,p): # loss function, kernel uses it to calculate loss - with tf.device(assign_device(p,'GPU')): # assign the device according to the process index p - return self.loss_object(labels,output) # return the mean softmax cross entropy loss between labels and output - - - def GradientTape(self,data,labels,p): - with tf.device(assign_device(p,'GPU')): - with tf.GradientTape(persistent=True) as tape: - output=self.fp(data,p) - loss=self.loss(output,labels,p) - return tape,output,loss - - - def opt(self,gradient,p): # optimization function, kernel uses it to optimize parameter - # Perform optimization on the parameters using the gradient - with tf.device(assign_device(p,'GPU')): # assign the device according to the process index p - param=self.optimizer(gradient,self.param) # apply the Note's momentum optimizer to update the parameters using the gradient - return param \ No newline at end of file diff --git a/7.0/neuralnetwork/RL/pytorch/non_parallel/DDPG.py b/7.0/neuralnetwork/RL/pytorch/non_parallel/DDPG.py deleted file mode 100644 index cf04915d..00000000 --- a/7.0/neuralnetwork/RL/pytorch/non_parallel/DDPG.py +++ /dev/null @@ -1,96 +0,0 @@ -import torch -import torch.nn.functional as F -import gym - - -class actor(torch.nn.Module): - def __init__(self,state_dim,hidden_dim,action_dim,action_bound): - super(actor,self).__init__() - self.fc1=torch.nn.Linear(state_dim,hidden_dim) - self.fc2=torch.nn.Linear(hidden_dim,action_dim) - self.action_bound=action_bound - - - def forward(self,x): - x=F.relu(self.fc1(x)) - return torch.tanh(self.fc2(x))*self.action_bound - - -class critic(torch.nn.Module): - def __init__(self,state_dim,hidden_dim,action_dim): - super(critic,self).__init__() - self.fc1=torch.nn.Linear(state_dim+action_dim,hidden_dim) - self.fc2=torch.nn.Linear(hidden_dim,1) - - - def forward(self,x,a): - cat=torch.cat([x,a],dim=1) - x=F.relu(self.fc1(cat)) - return self.fc2(x) - - -class DDPG: - def __init__(self,hidden_dim,sigma,gamma,tau,actor_lr,critic_lr): - if torch.cuda.is_available(): - self.device=torch.device('cuda') - else: - self.device=torch.device('cpu') - self.genv=gym.make('Pendulum-v1') - state_dim=self.genv.observation_space.shape[0] - action_dim=self.genv.action_space.shape[0] - action_bound=self.genv.action_space.high[0] - self.actor=actor(state_dim,hidden_dim,action_dim,action_bound).to(self.device) - self.critic=critic(state_dim,hidden_dim,action_dim).to(self.device) - self.target_actor=actor(state_dim,hidden_dim,action_dim,action_bound).to(self.device) - self.target_critic=critic(state_dim,hidden_dim,action_dim).to(self.device) - self.target_actor.load_state_dict(self.actor.state_dict()) - self.target_critic.load_state_dict(self.critic.state_dict()) - self.sigma=sigma - self.gamma=gamma - self.tau=tau - self.actor_opt=torch.optim.Adam(self.actor.parameters(),lr=actor_lr) - self.critic_opt=torch.optim.Adam(self.critic.parameters(),lr=critic_lr) - - - def env(self,a=None,initial=None): - if initial==True: - state=self.genv.reset(seed=0) - return state - else: - next_state,reward,done,_=self.genv.step(a) - return next_state,reward,done - - - def loss(self,s,a,next_s,r,d): - s=torch.tensor(s,dtype=torch.float).to(self.device) - a=torch.tensor(a,dtype=torch.float).view(-1,1).to(self.device) - next_s=torch.tensor(next_s,dtype=torch.float).to(self.device) - r=torch.tensor(r,dtype=torch.float).view(-1,1).to(self.device) - d=torch.tensor(d,dtype=torch.float).view(-1,1).to(self.device) - next_q_value=self.target_critic(next_s,self.target_actor(next_s)) - q_target=r+self.gamma*next_q_value*(1-d) - actor_loss=-torch.mean(self.critic(s,self.actor(s))) - critic_loss=F.mse_loss(self.critic(s,a),q_target) - return [actor_loss,critic_loss] - - - def backward(self,loss): - self.actor_opt.zero_grad() - loss[0].backward() - self.critic_opt.zero_grad() - loss[1].backward() - return - - - def opt(self): - self.actor_opt.step() - self.critic_opt.step() - return - - - def update_param(self): - for target_param,param in zip(self.target_actor.parameters(),self.actor.parameters()): - target_param.data.copy_(target_param.data*(1.0-self.tau)+param.data*self.tau) - for target_param,param in zip(self.target_critic.parameters(),self.critic.parameters()): - target_param.data.copy_(target_param.data*(1.0-self.tau)+param.data*self.tau) - return diff --git a/7.0/neuralnetwork/RL/pytorch/non_parallel/DQN.py b/7.0/neuralnetwork/RL/pytorch/non_parallel/DQN.py deleted file mode 100644 index a0e58689..00000000 --- a/7.0/neuralnetwork/RL/pytorch/non_parallel/DQN.py +++ /dev/null @@ -1,64 +0,0 @@ -import torch -import gym -import torch.nn.functional as F - - -class Qnet(torch.nn.Module): - def __init__(self,state_dim,hidden_dim,action_dim): - super(Qnet,self).__init__() - self.fc1=torch.nn.Linear(state_dim,hidden_dim) - self.fc2=torch.nn.Linear(hidden_dim,action_dim) - - - def forward(self,x): - x=F.relu(self.fc1(x)) - return self.fc2(x) - - -class DQN: - def __init__(self,state_dim,hidden_dim,action_dim): - if torch.cuda.is_available(): - self.device=torch.device('cuda') - else: - self.device=torch.device('cpu') - self.nn=Qnet(state_dim,hidden_dim,action_dim).to(self.device) - self.target_q_net=Qnet(state_dim,hidden_dim,action_dim).to(self.device) - self.optimizer=torch.optim.Adam(self.nn.parameters(),lr=2e-3) #optimizer,kernel uses it to optimize. - self.genv=gym.make('CartPole-v0') #create environment - - - def env(self,a=None,initial=None): #environment function,kernel uses it to interact with the environment. - if initial==True: - state=self.genv.reset() - return state - else: - next_state,reward,done,_=self.genv.step(a) - return next_state,reward,done - - - def loss(self,s,a,next_s,r,d): #loss function,kernel uses it to calculate loss. - s=torch.tensor(s,dtype=torch.float).to(self.device) - a=torch.tensor(a,dtype=torch.int64).view(-1,1).to(self.device) - next_s=torch.tensor(next_s,dtype=torch.float).to(self.device) - r=torch.tensor(r,dtype=torch.float).view(-1,1).to(self.device) - d=torch.tensor(d,dtype=torch.float).view(-1,1).to(self.device) - q_value=self.nn(s).gather(1,a) - next_q_value=self.target_q_net(next_s).max(1)[0].view(-1,1) - target=r+0.98*next_q_value*(1-d) - return F.mse_loss(q_value,target) - - - def backward(self,loss): #backward function,kernel uses it for backpropagation. - self.optimizer.zero_grad() - loss.backward() - return - - - def opt(self): #opt function,kernel uses it to optimize. - self.optimizer.step() - return - - - def update_param(self): #update function,kernel uses it to update parameter. - self.target_q_net.load_state_dict(self.nn.state_dict()) - return \ No newline at end of file diff --git a/7.0/neuralnetwork/RL/pytorch/non_parallel/DQN_pr.py b/7.0/neuralnetwork/RL/pytorch/non_parallel/DQN_pr.py deleted file mode 100644 index 66c71847..00000000 --- a/7.0/neuralnetwork/RL/pytorch/non_parallel/DQN_pr.py +++ /dev/null @@ -1,76 +0,0 @@ -import torch -import gym -import torch.nn.functional as F -import Note.create.RL.rl.prioritized_replay as pr - -#prioritized replay example -class Qnet(torch.nn.Module): - def __init__(self,state_dim,hidden_dim,action_dim): - super(Qnet,self).__init__() - self.fc1=torch.nn.Linear(state_dim,hidden_dim) - self.fc2=torch.nn.Linear(hidden_dim,action_dim) - - - def forward(self,x): - x=F.relu(self.fc1(x)) - return self.fc2(x) - - -class DQN: - def __init__(self,state_dim,hidden_dim,action_dim): - if torch.cuda.is_available(): - self.device=torch.device('cuda') - else: - self.device=torch.device('cpu') - self.nn=Qnet(state_dim,hidden_dim,action_dim).to(self.device) - self.target_q_net=Qnet(state_dim,hidden_dim,action_dim).to(self.device) - self.pr=pr.pr() - self.initial_TD=7 - self._epsilon=0.0007 - self.alpha=0.7 - self.optimizer=torch.optim.Adam(self.nn.parameters(),lr=2e-3) - self.genv=gym.make('CartPole-v0') - - - def env(self,a=None,initial=None): - if initial==True: - state=self.genv.reset(seed=0) - return state - else: - next_state,reward,done,_=self.genv.step(a) - return next_state,reward,done - - - def data_func(self,state_pool,action_pool,next_state_pool,reward_pool,done_pool,batch): - s,a,next_s,r,d=self.pr.sample(state_pool,action_pool,next_state_pool,reward_pool,done_pool,self.epsilon,self.alpha,batch) - return s,a,next_s,r,d - - - def loss(self,s,a,next_s,r,d): - s=torch.tensor(s,dtype=torch.float).to(self.device) - a=torch.tensor(a,dtype=torch.int64).view(-1,1).to(self.device) - next_s=torch.tensor(next_s,dtype=torch.float).to(self.device) - r=torch.tensor(r,dtype=torch.float).view(-1,1).to(self.device) - d=torch.tensor(d,dtype=torch.float).view(-1,1).to(self.device) - q_value=self.nn(s).gather(1,a) - next_q_value=self.target_q_net(next_s).max(1)[0].view(-1,1) - target=r+0.98*next_q_value*(1-d) - TD=target-q_value - self.pr.update_TD(TD) - return torch.mean(TD**2) - - - def backward(self,loss): - self.optimizer.zero_grad() - loss.backward() - return - - - def opt(self): - self.optimizer.step() - return - - - def update_param(self): - self.target_q_net.load_state_dict(self.nn.state_dict()) - return \ No newline at end of file diff --git a/7.0/neuralnetwork/RL/pytorch/non_parallel/DoubleDQN.py b/7.0/neuralnetwork/RL/pytorch/non_parallel/DoubleDQN.py deleted file mode 100644 index 31e606e8..00000000 --- a/7.0/neuralnetwork/RL/pytorch/non_parallel/DoubleDQN.py +++ /dev/null @@ -1,65 +0,0 @@ -import torch -import gym -import torch.nn.functional as F - - -class Qnet(torch.nn.Module): - def __init__(self,state_dim,hidden_dim,action_dim): - super(Qnet,self).__init__() - self.fc1=torch.nn.Linear(state_dim,hidden_dim) - self.fc2=torch.nn.Linear(hidden_dim,action_dim) - - - def forward(self,x): - x=F.relu(self.fc1(x)) - return self.fc2(x) - - -class DoubleDQN: - def __init__(self,state_dim,hidden_dim,action_dim): - if torch.cuda.is_available(): - self.device=torch.device('cuda') - else: - self.device=torch.device('cpu') - self.nn=Qnet(state_dim,hidden_dim,action_dim).to(self.device) - self.target_q_net=Qnet(state_dim,hidden_dim,action_dim).to(self.device) - self.optimizer=torch.optim.Adam(self.nn.parameters(),lr=2e-3) - self.genv=gym.make('CartPole-v0') - - - def env(self,a=None,initial=None): - if initial==True: - state=self.genv.reset(seed=0) - return state - else: - next_state,reward,done,_=self.genv.step(a) - return next_state,reward,done - - - def loss(self,s,a,next_s,r,d): - s=torch.tensor(s,dtype=torch.float).to(self.device) - a=torch.tensor(a,dtype=torch.int64).view(-1,1).to(self.device) - next_s=torch.tensor(next_s,dtype=torch.float).to(self.device) - r=torch.tensor(r,dtype=torch.float).view(-1,1).to(self.device) - d=torch.tensor(d,dtype=torch.float).view(-1,1).to(self.device) - q_value=self.nn(s).gather(1,a) - max_action=self.nn(next_s).max(1)[1].view(-1,1) - next_q_value=self.target_q_net(next_s).gather(1,max_action) - target=r+0.98*next_q_value*(1-d) - return F.mse_loss(q_value,target) - - - def backward(self,loss): - self.optimizer.zero_grad() - loss.backward() - return - - - def opt(self): - self.optimizer.step() - return - - - def update_param(self): - self.target_q_net.load_state_dict(self.nn.state_dict()) - return \ No newline at end of file diff --git a/7.0/neuralnetwork/RL/pytorch/non_parallel/DuelingDQN.py b/7.0/neuralnetwork/RL/pytorch/non_parallel/DuelingDQN.py deleted file mode 100644 index b81eeb37..00000000 --- a/7.0/neuralnetwork/RL/pytorch/non_parallel/DuelingDQN.py +++ /dev/null @@ -1,67 +0,0 @@ -import torch -import gym -import torch.nn.functional as F - - -class VAnet(torch.nn.Module): - def __init__(self,state_dim,hidden_dim,action_dim): - super(VAnet,self).__init__() - self.fc1=torch.nn.Linear(state_dim,hidden_dim) - self.fc_A=torch.nn.Linear(hidden_dim,action_dim) - self.fc_V=torch.nn.Linear(hidden_dim,1) - - - def forward(self,x): - A=self.fc_A(F.relu(self.fc1(x))) - V=self.fc_V(F.relu(self.fc1(x))) - Q=V+A-A.mean(1).view(-1,1) - return Q - - -class DuelingDQN: - def __init__(self,state_dim,hidden_dim,action_dim): - if torch.cuda.is_available(): - self.device=torch.device('cuda') - else: - self.device=torch.device('cpu') - self.nn=VAnet(state_dim,hidden_dim,action_dim).to(self.device) - self.target_q_net=VAnet(state_dim,hidden_dim,action_dim).to(self.device) - self.optimizer=torch.optim.Adam(self.nn.parameters(),lr=2e-3) - self.genv=gym.make('CartPole-v0') - - - def env(self,a=None,initial=None): - if initial==True: - state=self.genv.reset(seed=0) - return state - else: - next_state,reward,done,_=self.genv.step(a) - return next_state,reward,done - - - def loss(self,s,a,next_s,r,d): - s=torch.tensor(s,dtype=torch.float).to(self.device) - a=torch.tensor(a,dtype=torch.int64).view(-1,1).to(self.device) - next_s=torch.tensor(next_s,dtype=torch.float).to(self.device) - r=torch.tensor(r,dtype=torch.float).view(-1,1).to(self.device) - d=torch.tensor(d,dtype=torch.float).view(-1,1).to(self.device) - q_value=self.nn(s).gather(1,a) - next_q_value=self.target_q_net(next_s).max(1)[0].view(-1,1) - target=r+0.98*next_q_value*(1-d) - return F.mse_loss(q_value,target) - - - def backward(self,loss): - self.optimizer.zero_grad() - loss.backward() - return - - - def opt(self): - self.optimizer.step() - return - - - def update_param(self): - self.target_q_net.load_state_dict(self.nn.state_dict()) - return \ No newline at end of file diff --git a/7.0/neuralnetwork/RL/pytorch/parallel/DQN.py b/7.0/neuralnetwork/RL/pytorch/parallel/DQN.py deleted file mode 100644 index f4d8417d..00000000 --- a/7.0/neuralnetwork/RL/pytorch/parallel/DQN.py +++ /dev/null @@ -1,65 +0,0 @@ -import torch -import gym -import torch.nn.functional as F -from Note.nn.parallel.assign_device_pytorch import assign_device - - -class Qnet(torch.nn.Module): - def __init__(self,state_dim,hidden_dim,action_dim): - super(Qnet,self).__init__() - self.fc1=torch.nn.Linear(state_dim,hidden_dim) - self.fc2=torch.nn.Linear(hidden_dim,action_dim) - - - def forward(self,x): - x=F.relu(self.fc1(x)) - return self.fc2(x) - - -class DQN: - def __init__(self,state_dim,hidden_dim,action_dim): - if torch.cuda.is_available(): - self.device=torch.device('cuda') - else: - self.device=torch.device('cpu') - self.nn=Qnet(state_dim,hidden_dim,action_dim).to(self.device) - self.target_q_net=Qnet(state_dim,hidden_dim,action_dim).to(self.device) - self.optimizer=[torch.optim.Adam(self.nn.parameters(),lr=2e-3) for _ in range(5)] #optimizer,kernel uses it to optimize. - self.genv=[gym.make('CartPole-v0') for _ in range(5)] #create environment - - - def env(self,a=None,p=None,initial=None): #environment function,kernel uses it to interact with the environment. - if initial==True: - state=self.genv[p].reset() - return state - else: - next_state,reward,done,_=self.genv[p].step(a) - return next_state,reward,done - - - def loss(self,s,a,next_s,r,d,p): #loss function,kernel uses it to calculate loss. - s=torch.tensor(s,dtype=torch.float).to(assign_device(p,'GPU')) - a=torch.tensor(a,dtype=torch.int64).view(-1,1).to(assign_device(p,'GPU')) - next_s=torch.tensor(next_s,dtype=torch.float).to(assign_device(p,'GPU')) - r=torch.tensor(r,dtype=torch.float).view(-1,1).to(assign_device(p,'GPU')) - d=torch.tensor(d,dtype=torch.float).view(-1,1).to(assign_device(p,'GPU')) - q_value=self.nn(s).gather(1,a) - next_q_value=self.target_q_net(next_s).max(1)[0].view(-1,1) - target=r+0.98*next_q_value*(1-d) - return F.mse_loss(q_value,target) - - - def backward(self,loss,p): #backward function,kernel uses it for backpropagation. - self.optimizer[p].zero_grad(set_to_none=True) - loss.backward() - return - - - def opt(self,p): #opt function,kernel uses it to optimize. - self.optimizer[p].step() - return - - - def update_param(self): #update function,kernel uses it to update parameter. - self.target_q_net.load_state_dict(self.nn.state_dict()) - return \ No newline at end of file diff --git a/7.0/neuralnetwork/RL/tensorflow/non_parallel/DDPG.py b/7.0/neuralnetwork/RL/tensorflow/non_parallel/DDPG.py deleted file mode 100644 index 2f4aa1e2..00000000 --- a/7.0/neuralnetwork/RL/tensorflow/non_parallel/DDPG.py +++ /dev/null @@ -1,76 +0,0 @@ -import tensorflow as tf # import TensorFlow library -from Note import nn -import gym # import OpenAI Gym library - - -class actor: # define a class for the actor network - def __init__(self,state_dim,hidden_dim,action_dim,action_bound): # initialize the network with state dimension, hidden dimension, action dimension and action bound - self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') - self.dense2 = nn.dense(action_dim, hidden_dim, activation='tanh') - self.action_bound=action_bound - self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list - - - def fp(self,x): # forward propagation function, kernel uses it for forward propagation - x = self.dense1(x) - return self.dense2(x)*self.action_bound - - -class critic: # define a class for the critic network - def __init__(self,state_dim,hidden_dim,action_dim): # initialize the network with state dimension, hidden dimension and action dimension - self.dense1 = nn.dense(hidden_dim, state_dim+action_dim, activation='relu') - self.dense2 = nn.dense(action_dim, hidden_dim) - self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list - - - def fp(self,x,a): # forward propagation function, kernel uses it for forward propagation - cat=tf.concat([x,a],axis=1) - x=self.dense1(cat) - return self.dense2(x) - - -class DDPG: # define a class for the DDPG agent - def __init__(self,hidden_dim,sigma,gamma,tau,actor_lr,critic_lr): # initialize the agent with hidden dimension, noise scale, discount factor, soft update factor, actor learning rate and critic learning rate - self.genv=gym.make('Pendulum-v1') # create environment - state_dim=self.genv.observation_space.shape[0] # get the state dimension from the environment observation space - action_dim=self.genv.action_space.shape[0] # get the action dimension from the environment action space - action_bound=self.genv.action_space.high[0] # get the action bound from the environment action space high value (assume symmetric) - self.actor=actor(state_dim,hidden_dim,action_dim,action_bound) # create an actor network for the agent - self.critic=critic(state_dim,hidden_dim,action_dim) # create a critic network for the agent - self.target_actor=actor(state_dim,hidden_dim,action_dim,action_bound) # create a target actor network for the agent - self.target_critic=critic(state_dim,hidden_dim,action_dim) # create a target critic network for the agent - self.actor_param=self.actor.param # parameter list of actor network, kernel uses it list for backpropagation - self.critic_param=self.critic.param # parameter list of critic network, kernel uses it list for backpropagation - nn.assign_param(self.target_actor.param,self.actor_param.copy()) # copy the parameters from actor network to target actor network - nn.assign_param(self.target_critic.param,self.critic_param.copy()) # copy the parameters from critic network to target critic network - self.param=[self.actor_param,self.critic_param] # parameter list of both networks, kernel uses it list for backpropagation - self.sigma=sigma # noise scale - self.gamma=gamma # discount factor - self.tau=tau # soft update factor - self.opt=tf.keras.optimizers.Adam() # optimizer, kernel uses it to optimize. Here we use Adam optimizer - - - def env(self,a=None,initial=None): # environment function, kernel uses it to interact with the environment - if initial==True: # if initial is True, reset the environment and return the initial state - state=self.genv.reset(seed=0) - return state - else: # otherwise, take an action and return the next state, reward and done flag - next_state,reward,done,_=self.genv.step(a) - return next_state,reward,done - - - def loss(self,s,a,next_s,r,d): # loss function, kernel uses it to calculate loss - a=tf.expand_dims(a,axis=1) # expand the action vector to match the Q-value matrix shape - next_q_value=self.target_critic.fp(next_s,self.target_actor.fp(next_s)) # get the Q-value for the next state and action from the target critic network - q_target=tf.cast(r,'float32')+self.gamma*next_q_value*(1-tf.cast(d,'float32')) # calculate the target value using Bellman equation with discount factor gamma - actor_loss=-tf.reduce_mean(self.critic.fp(s,self.actor.fp(s))) # calculate the actor loss as the negative mean of the Q-value for the current state and action from the critic network - critic_loss=tf.reduce_mean((self.critic.fp(s,a)-q_target)**2) # calculate the critic loss as the mean squared error between Q-value and target value - return actor_loss+critic_loss # return a list of actor loss and critic loss - - - def update_param(self): # update function, kernel uses it to update parameter - for target_param,param in zip(self.target_actor.param,self.actor.param): # for each pair of parameters in target actor network and actor network - target_param.assign(target_param*(1.0-self.tau)+param*self.tau) # update the target parameter using soft update with factor tau - for target_param,param in zip(self.target_critic.param,self.critic.param): # for each pair of parameters in target critic network and critic network - target_param.assign(target_param*(1.0-self.tau)+param*self.tau) # update the target parameter using soft update with factor tau - return diff --git a/7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN.py b/7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN.py deleted file mode 100644 index d0be2840..00000000 --- a/7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN.py +++ /dev/null @@ -1,45 +0,0 @@ -import tensorflow as tf # import TensorFlow library -from Note import nn -import gym # import OpenAI Gym library - - -class Qnet: # define a class for the Q-network - def __init__(self,state_dim,hidden_dim,action_dim): # initialize the network with state dimension, hidden dimension and action dimension - self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') - self.dense2 = nn.dense(action_dim, hidden_dim) - self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list - - def fp(self,x): # forward propagation function, kernel uses it for forward propagation - x = self.dense2(self.dense1(x)) - return x - - -class DQN: # define a class for the DQN agent - def __init__(self,state_dim,hidden_dim,action_dim): # initialize the agent with state dimension, hidden dimension and action dimension - self.nn=Qnet(state_dim,hidden_dim,action_dim) # create a Q-network for the agent - self.target_q_net=Qnet(state_dim,hidden_dim,action_dim) # create a target Q-network for the agent - self.param=self.nn.param # parameter list, kernel uses it list for backpropagation - self.opt=tf.keras.optimizers.Adam() # optimizer, kernel uses it to optimize - self.genv=gym.make('CartPole-v0') # create environment - - - def env(self,a=None,initial=None): # environment function, kernel uses it to interact with the environment - if initial==True: # if initial is True, reset the environment and return the initial state - state=self.genv.reset(seed=0) - return state - else: # otherwise, take an action and return the next state, reward and done flag - next_state,reward,done,_=self.genv.step(a) - return next_state,reward,done - - - def loss(self,s,a,next_s,r,d): # loss function, kernel uses it to calculate loss - a=tf.expand_dims(a,axis=1) # expand the action vector to match the Q-value matrix shape - q_value=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) # get the Q-value for the selected action - next_q_value=tf.reduce_max(self.target_q_net.fp(next_s),axis=1) # get the maximum Q-value for the next state from the target network - target=tf.cast(r,'float32')+0.98*next_q_value*(1-tf.cast(d,'float32')) # calculate the target value using Bellman equation with discount factor 0.98 - return tf.reduce_mean((q_value-target)**2) # return the mean squared error between Q-value and target value - - - def update_param(self): # update function, kernel uses it to update parameter - nn.assign_param(self.target_q_net.param, self.param) # copy the parameters from the Q-network to the target network - return \ No newline at end of file diff --git a/7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN_pr.py b/7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN_pr.py deleted file mode 100644 index e907903a..00000000 --- a/7.0/neuralnetwork/RL/tensorflow/non_parallel/DQN_pr.py +++ /dev/null @@ -1,58 +0,0 @@ -import tensorflow as tf -from Note import nn -import gym -import Note.RL.rl.prioritized_replay as pr - - -class Qnet: - def __init__(self,state_dim,hidden_dim,action_dim): - self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') - self.dense2 = nn.dense(action_dim, hidden_dim) - self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list - - def fp(self,x): - x = self.dense2(self.dense1(x)) - return x - - -class DQN: - def __init__(self,state_dim,hidden_dim,action_dim): - self.nn=Qnet(state_dim,hidden_dim,action_dim) - self.target_q_net=Qnet(state_dim,hidden_dim,action_dim) - self.param=self.nn.param - self.pr=pr.pr() - self.initial_TD=7 - self.epsilon=0.0007 - self.alpha=0.7 - self.opt=tf.keras.optimizers.Adam() - self.genv=gym.make('CartPole-v0') - - - def env(self,a=None,initial=None): - if initial==True: - state=self.genv.reset(seed=0) - return state - else: - next_state,reward,done,_=self.genv.step(a) - return next_state,reward,done - - - def data_func(self,state_pool,action_pool,next_state_pool,reward_pool,done_pool,batch): - s,a,next_s,r,d=self.pr.sample(state_pool,action_pool,next_state_pool,reward_pool,done_pool,self.epsilon,self.alpha,batch) - return s,a,next_s,r,d - - - def loss(self,s,a,next_s,r,d): - a=tf.expand_dims(a,axis=1) - q_value=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) - next_q_value=tf.reduce_max(self.target_q_net.fp(next_s),axis=1) - target=tf.cast(r,'float32')+0.98*next_q_value*(1-tf.cast(d,'float32')) - target=tf.expand_dims(target,axis=1) - TD=target-q_value - self.pr.update_TD(TD) - return tf.reduce_mean(TD**2) - - - def update_param(self): - nn.assign_param(self.target_q_net.param, self.param) - return \ No newline at end of file diff --git a/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py b/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py deleted file mode 100644 index c0e9b704..00000000 --- a/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO.py +++ /dev/null @@ -1,68 +0,0 @@ -import tensorflow as tf -from Note import nn -import gym - - -class actor: - def __init__(self,state_dim,hidden_dim,action_dim): - self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') - self.dense2 = nn.dense(action_dim, hidden_dim) - self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list - - def fp(self,x): - x=self.dense1(x) - return tf.nn.softmax(self.dense2(x)) - - -class critic: - def __init__(self,state_dim,hidden_dim): - self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') - self.dense2 = nn.dense(1, hidden_dim) - self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list - - def fp(self,x): - x=self.dense1(x) - return self.dense2(x) - - -class PPO: - def __init__(self,state_dim,hidden_dim,action_dim,clip_eps,alpha): - self.actor=actor(state_dim,hidden_dim,action_dim) - self.nn=actor(state_dim,hidden_dim,action_dim) - nn.assign_param(self.nn.param,self.actor.param.copy()) - self.critic=critic(state_dim,hidden_dim) - self.clip_eps=clip_eps - self.alpha=alpha - self.param=[self.actor.param,self.critic.param] - self.opt=tf.keras.optimizers.Adam() - self.genv=gym.make('CartPole-v0') - - - def env(self,a=None,initial=None): - if initial==True: - state=self.genv.reset(seed=0) - return state - else: - next_state,reward,done,_=self.genv.step(a) - return next_state,reward,done - - - def loss(self,s,a,next_s,r,d): - a=tf.expand_dims(a,axis=1) - action_prob=tf.gather(self.actor.fp(s),a,axis=1,batch_dims=1) - action_prob_old=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) - raito=action_prob/action_prob_old - value=self.critic.fp(s) - value_tar=tf.cast(r,'float32')+0.98*self.critic.fp(next_s)*(1-tf.cast(d,'float32')) - TD=value_tar-value - sur1=raito*TD - sur2=tf.clip_by_value(raito,clip_value_min=1-self.clip_eps,clip_value_max=1+self.clip_eps)*TD - clip_loss=-tf.math.minimum(sur1,sur2) - entropy=action_prob*tf.math.log(action_prob+1e-8) - clip_loss=clip_loss-self.alpha*entropy - return tf.reduce_mean(clip_loss)+tf.reduce_mean((TD)**2) - - - def update_param(self): - nn.assign_param(self.nn.param, self.actor.param) - return \ No newline at end of file diff --git a/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO_eps.py b/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO_eps.py deleted file mode 100644 index 1287521b..00000000 --- a/7.0/neuralnetwork/RL/tensorflow/non_parallel/PPO_eps.py +++ /dev/null @@ -1,63 +0,0 @@ -import tensorflow as tf -from Note import nn -import gym - - -class actor: - def __init__(self,state_dim,hidden_dim,action_dim): - self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') - self.dense2 = nn.dense(action_dim, hidden_dim) - self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list - - def fp(self,x): - x=self.dense1(x) - return tf.nn.softmax(self.dense2(x)) - - -class critic: - def __init__(self,state_dim,hidden_dim): - self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') - self.dense2 = nn.dense(1, hidden_dim) - self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list - - def fp(self,x): - x=self.dense1(x) - return self.dense2(x) - - -class PPO: - def __init__(self,state_dim,hidden_dim,action_dim,clip_eps): - self.actor=actor(state_dim,hidden_dim,action_dim) - self.nn=actor(state_dim,hidden_dim,action_dim) - nn.assign_param(self.nn.param,self.actor.param.copy()) - self.critic=critic(state_dim,hidden_dim) - self.clip_eps=clip_eps - self.param=[self.actor.param,self.critic.param] - self.opt=tf.keras.optimizers.Adam() - self.genv=gym.make('CartPole-v0') - - - def env(self,a=None,initial=None): - if initial==True: - state=self.genv.reset(seed=0) - return state - else: - next_state,reward,done,_=self.genv.step(a) - return next_state,reward,done - - - def loss(self,s,a,next_s,r,d): - a=tf.expand_dims(a,axis=1) - raito=tf.gather(self.actor.fp(s),a,axis=1,batch_dims=1)/tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) - value=self.critic.fp(s) - value_tar=tf.cast(r,'float32')+0.98*self.critic.fp(next_s)*(1-tf.cast(d,'float32')) - TD=value_tar-value - sur1=raito*TD - sur2=tf.clip_by_value(raito,clip_value_min=1-self.clip_eps,clip_value_max=1+self.clip_eps)*TD - clip_loss=-tf.math.minimum(sur1,sur2) - return tf.reduce_mean(clip_loss)+tf.reduce_mean((TD)**2) - - - def update_param(self): - nn.assign_param(self.nn.param, self.actor.param) - return \ No newline at end of file diff --git a/7.0/neuralnetwork/RL/tensorflow/parallel/DDPG.py b/7.0/neuralnetwork/RL/tensorflow/parallel/DDPG.py deleted file mode 100644 index 4235f50e..00000000 --- a/7.0/neuralnetwork/RL/tensorflow/parallel/DDPG.py +++ /dev/null @@ -1,77 +0,0 @@ -import tensorflow as tf # import TensorFlow library -from Note import nn -import gym # import OpenAI Gym library -import Note.nn.parallel.optimizer as o # import Note's optimizer module - - -class actor: # define a class for the actor network - def __init__(self,state_dim,hidden_dim,action_dim,action_bound): # initialize the network with state dimension, hidden dimension, action dimension and action bound - self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') - self.dense2 = nn.dense(action_dim, hidden_dim, activation='tanh') - self.action_bound=action_bound - self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list - - - def fp(self,x): # forward propagation function, kernel uses it for forward propagation - x = self.dense1(x) - return self.dense2(x)*self.action_bound - - -class critic: # define a class for the critic network - def __init__(self,state_dim,hidden_dim,action_dim): # initialize the network with state dimension, hidden dimension and action dimension - self.dense1 = nn.dense(hidden_dim, state_dim+action_dim, activation='relu') - self.dense2 = nn.dense(action_dim, hidden_dim) - self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list - - - def fp(self,x,a): # forward propagation function, kernel uses it for forward propagation - cat=tf.concat([x,a],axis=1) - x=self.dense1(cat) - return self.dense2(x) - - -class DDPG: # define a class for the DDPG agent - def __init__(self,hidden_dim,sigma,gamma,tau,actor_lr,critic_lr): # initialize the agent with hidden dimension, noise scale, discount factor, soft update factor, actor learning rate and critic learning rate - self.genv=[gym.make('Pendulum-v1') for _ in range(5)] # create a list of 5 environments - state_dim=self.genv.observation_space.shape[0] # get the state dimension from the environment observation space - action_dim=self.genv.action_space.shape[0] # get the action dimension from the environment action space - action_bound=self.genv.action_space.high[0] # get the action bound from the environment action space high value (assume symmetric) - self.actor=actor(state_dim,hidden_dim,action_dim,action_bound) # create an actor network for the agent - self.critic=critic(state_dim,hidden_dim,action_dim) # create a critic network for the agent - self.target_actor=actor(state_dim,hidden_dim,action_dim,action_bound) # create a target actor network for the agent - self.target_critic=critic(state_dim,hidden_dim,action_dim) # create a target critic network for the agent - self.actor_param=self.actor.param # parameter list of actor network, kernel uses it list for backpropagation - self.critic_param=self.critic.param # parameter list of critic network, kernel uses it list for backpropagation - nn.assign_param(self.target_actor.param,self.actor_param.copy()) # copy the parameters from actor network to target actor network - nn.assign_param(self.target_critic.param,self.critic_param.copy()) # copy the parameters from critic network to target critic network - self.param=[self.actor_param,self.critic_param] # parameter list of both networks, kernel uses it list for backpropagation - self.sigma=sigma # noise scale - self.gamma=gamma # discount factor - self.tau=tau # soft update factor - self.optimizer=o.SGD(param=self.param) # optimizer, kernel uses it to optimize. Here we use a custom SGD optimizer - - - def env(self,a=None,p=None,initial=None): # environment function, kernel uses it to interact with the environment - if initial==True: # if initial is True, reset the environment with index p and return the initial state - state=self.genv[p].reset(seed=0) - return state - else: # otherwise, take an action and return the next state, reward and done flag from the environment with index p - next_state,reward,done,_=self.genv[p].step(a) - return next_state,reward,done - - - def loss(self,s,a,next_s,r,d): # loss function, kernel uses it to calculate loss - a=tf.expand_dims(a,axis=1) # expand the action vector to match the Q-value matrix shape - next_q_value=self.target_critic.fp(next_s,self.target_actor.fp(next_s)) # get the Q-value for the next state and action from the target critic network - q_target=tf.cast(r,'float32')+self.gamma*next_q_value*(1-tf.cast(d,'float32')) # calculate the target value using Bellman equation with discount factor gamma - actor_loss=-tf.reduce_mean(self.critic.fp(s,self.actor.fp(s))) # calculate the actor loss as the negative mean of the Q-value for the current state and action from the critic network - critic_loss=tf.reduce_mean((self.critic.fp(s,a)-q_target)**2) # calculate the critic loss as the mean squared error between Q-value and target value - return actor_loss+critic_loss # return a list of actor loss and critic loss - - - def update_param(self): # update function, kernel uses it to update parameter - for target_param,param in zip(self.target_actor.param,self.actor.param): # for each pair of parameters in target actor network and actor network - target_param.assign(target_param*(1.0-self.tau)+param*self.tau) # update the target parameter using soft update with factor tau - for target_param,param in zip(self.target_critic.param,self.critic.param): # for each pair of parameters in target critic network and critic network - target_param.assign(target_param*(1.0-self.tau)+param*self.tau) # update the target parameter using soft update with factor tau - return diff --git a/7.0/neuralnetwork/RL/tensorflow/parallel/DQN.py b/7.0/neuralnetwork/RL/tensorflow/parallel/DQN.py deleted file mode 100644 index 9186307d..00000000 --- a/7.0/neuralnetwork/RL/tensorflow/parallel/DQN.py +++ /dev/null @@ -1,51 +0,0 @@ -import tensorflow as tf # import TensorFlow library -from Note import nn -import gym # import OpenAI Gym library -import Note.nn.parallel.optimizer as o # import Note's optimizer module - - -class Qnet: # define a class for the Q-network - def __init__(self,state_dim,hidden_dim,action_dim): # initialize the network with state dimension, hidden dimension and action dimension - self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') - self.dense2 = nn.dense(action_dim, hidden_dim) - self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list - - def fp(self,x): # forward propagation function, kernel uses it for forward propagation - x = self.dense2(self.dense1(x)) - return x - - -class DQN: # define a class for the DQN agent - def __init__(self,state_dim,hidden_dim,action_dim): # initialize the agent with state dimension, hidden dimension and action dimension - self.nn=Qnet(state_dim,hidden_dim,action_dim) # create a Q-network for the agent - self.target_q_net=Qnet(state_dim,hidden_dim,action_dim) # create a target Q-network for the agent - self.param=self.nn.param # parameter list, kernel uses it list for backpropagation - self.optimizer=o.SGD(param=self.param) # optimizer, kernel uses it to optimize. Here we use a custom SGD optimizer - self.genv=[gym.make('CartPole-v0') for _ in range(5)] # create a list of 5 environments - - - def env(self,a=None,p=None,initial=None): # environment function, kernel uses it to interact with the environment - if initial==True: # if initial is True, reset the environment with index p and return the initial state - state=self.genv[p].reset(seed=0) - return state - else: # otherwise, take an action and return the next state, reward and done flag from the environment with index p - next_state,reward,done,_=self.genv[p].step(a) - return next_state,reward,done - - - def loss(self,s,a,next_s,r,d): # loss function, kernel uses it to calculate loss - a=tf.expand_dims(a,axis=1) # expand the action vector to match the Q-value matrix shape - q_value=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) # get the Q-value for the selected action - next_q_value=tf.reduce_max(self.target_q_net.fp(next_s),axis=1) # get the maximum Q-value for the next state from the target network - target=tf.cast(r,'float32')+0.98*next_q_value*(1-tf.cast(d,'float32')) # calculate the target value using Bellman equation with discount factor 0.98 - return tf.reduce_mean((q_value-target)**2) # return the mean squared error between Q-value and target value - - - def update_param(self): # update function, kernel uses it to update parameter - nn.assign(self.target_q_net.param,self.param) # copy the parameters from the Q-network to the target network - return - - - def opt(self,gradient): # optimization function, kernel uses it to optimize parameter - param=self.optimizer.opt(gradient,self.param) # apply the custom momentum optimizer to update the parameters using the gradient - return param \ No newline at end of file diff --git a/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py b/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py deleted file mode 100644 index 88763150..00000000 --- a/7.0/neuralnetwork/RL/tensorflow/parallel/PPO.py +++ /dev/null @@ -1,68 +0,0 @@ -import tensorflow as tf -from Note import nn -import gym -from Note.nn.parallel.optimizer import SGD - - -class actor: - def __init__(self,state_dim,hidden_dim,action_dim): - self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') - self.dense2 = nn.dense(action_dim, hidden_dim) - self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list - - def fp(self,x): - x=self.dense1(x) - return tf.nn.softmax(self.dense2(x)) - - -class critic: - def __init__(self,state_dim,hidden_dim): - self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') - self.dense2 = nn.dense(1, hidden_dim) - self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list - - def fp(self,x): - x=self.dense1(x) - return self.dense2(x) - - -class PPO: - def __init__(self,state_dim,hidden_dim,action_dim,clip_eps,alpha): - self.actor=actor(state_dim,hidden_dim,action_dim) - self.nn=actor(state_dim,hidden_dim,action_dim) - nn.assign(self.nn.param,self.actor.param.copy()) - self.critic=critic(state_dim,hidden_dim) - self.clip_eps=clip_eps - self.param=[self.actor.param,self.critic.param] - self.opt=SGD(param=self.param) - self.genv=[gym.make('CartPole-v0') for _ in range(5)] - - - def env(self,a=None,p=None,initial=None): - if initial==True: - state=self.genv[p].reset(seed=0) - return state - else: - next_state,reward,done,_=self.genv[p].step(a) - return next_state,reward,done - - - def loss(self,s,a,next_s,r,d): - a=tf.expand_dims(a,axis=1) - action_prob=tf.gather(self.actor.fp(s),a,axis=1,batch_dims=1) - action_prob_old=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) - raito=action_prob/action_prob_old - value=self.critic.fp(s) - value_tar=tf.cast(r,'float32')+0.98*self.critic.fp(next_s)*(1-tf.cast(d,'float32')) - TD=value_tar-value - sur1=raito*TD - sur2=tf.clip_by_value(raito,clip_value_min=1-self.clip_eps,clip_value_max=1+self.clip_eps)*TD - clip_loss=-tf.math.minimum(sur1,sur2) - entropy=action_prob*tf.math.log(action_prob+1e-8) - clip_loss=clip_loss-self.alpha*entropy - return tf.reduce_mean(clip_loss)+tf.reduce_mean((TD)**2) - - - def update_param(self): - nn.assign(self.nn.param,self.actor.param) - return \ No newline at end of file diff --git a/7.0/neuralnetwork/RL/tensorflow/parallel/PPO_eps.py b/7.0/neuralnetwork/RL/tensorflow/parallel/PPO_eps.py deleted file mode 100644 index 008baf04..00000000 --- a/7.0/neuralnetwork/RL/tensorflow/parallel/PPO_eps.py +++ /dev/null @@ -1,64 +0,0 @@ -import tensorflow as tf -from Note import nn -import gym -from Note.nn.parallel.optimizer import SGD - - -class actor: - def __init__(self,state_dim,hidden_dim,action_dim): - self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') - self.dense2 = nn.dense(action_dim, hidden_dim) - self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list - - def fp(self,x): - x=self.dense1(x) - return tf.nn.softmax(self.dense2(x)) - - -class critic: - def __init__(self,state_dim,hidden_dim): - self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') - self.dense2 = nn.dense(1, hidden_dim) - self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list - - def fp(self,x): - x=self.dense1(x) - return self.dense2(x) - - -class PPO: - def __init__(self,state_dim,hidden_dim,action_dim,clip_eps): - self.actor=actor(state_dim,hidden_dim,action_dim) - self.nn=actor(state_dim,hidden_dim,action_dim) - nn.assign(self.nn.param,self.actor.param.copy()) - self.critic=critic(state_dim,hidden_dim) - self.clip_eps=clip_eps - self.param=[self.actor.param,self.critic.param] - self.opt=SGD(param=self.param) - self.genv=[gym.make('CartPole-v0') for _ in range(5)] - - - def env(self,a=None,p=None,initial=None): - if initial==True: - state=self.genv[p].reset(seed=0) - return state - else: - next_state,reward,done,_=self.genv[p].step(a) - return next_state,reward,done - - - def loss(self,s,a,next_s,r,d): - a=tf.expand_dims(a,axis=1) - raito=tf.gather(self.actor.fp(s),a,axis=1,batch_dims=1)/tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) - value=self.critic.fp(s) - value_tar=tf.cast(r,'float32')+0.98*self.critic.fp(next_s)*(1-tf.cast(d,'float32')) - TD=value_tar-value - sur1=raito*TD - sur2=tf.clip_by_value(raito,clip_value_min=1-self.clip_eps,clip_value_max=1+self.clip_eps)*TD - clip_loss=-tf.math.minimum(sur1,sur2) - return tf.reduce_mean(clip_loss)+tf.reduce_mean((TD)**2) - - - def update_param(self): - nn.assign(self.nn.param,self.actor.param) - return \ No newline at end of file From 4efed67dd0bf0db5a9d45a124e6a0b454278eba2 Mon Sep 17 00:00:00 2001 From: NoteDance Date: Tue, 6 Aug 2024 09:49:16 +0800 Subject: [PATCH 316/337] Add files via upload --- 7.0/models/DL/pytorch/non_parallel/nn.py | 53 ++++++++++ 7.0/models/DL/pytorch/parallel/nn.py | 41 ++++++++ 7.0/models/DL/pytorch/parallel/nn_device.py | 45 +++++++++ 7.0/models/DL/tensorflow/non_parallel/cnn.py | 27 ++++++ .../DL/tensorflow/non_parallel/layer/cnn.py | 48 ++++++++++ .../DL/tensorflow/non_parallel/layer/nn.py | 38 ++++++++ 7.0/models/DL/tensorflow/non_parallel/nn.py | 23 +++++ .../DL/tensorflow/non_parallel/nn_acc.py | 28 ++++++ .../DL/tensorflow/non_parallel/nn_ol.py | 37 +++++++ 7.0/models/DL/tensorflow/parallel/cnn.py | 51 ++++++++++ 7.0/models/DL/tensorflow/parallel/nn.py | 41 ++++++++ 7.0/models/DL/tensorflow/parallel/nn_acc.py | 46 +++++++++ .../DL/tensorflow/parallel/nn_attenuate.py | 56 +++++++++++ .../DL/tensorflow/parallel/nn_clipping.py | 49 ++++++++++ .../DL/tensorflow/parallel/nn_device.py | 50 ++++++++++ 7.0/models/RL/pytorch/non_parallel/DDPG.py | 96 +++++++++++++++++++ 7.0/models/RL/pytorch/non_parallel/DQN.py | 64 +++++++++++++ 7.0/models/RL/pytorch/non_parallel/DQN_pr.py | 76 +++++++++++++++ .../RL/pytorch/non_parallel/DoubleDQN.py | 65 +++++++++++++ .../RL/pytorch/non_parallel/DuelingDQN.py | 67 +++++++++++++ 7.0/models/RL/pytorch/parallel/DQN.py | 65 +++++++++++++ 7.0/models/RL/tensorflow/non_parallel/DDPG.py | 76 +++++++++++++++ 7.0/models/RL/tensorflow/non_parallel/DQN.py | 45 +++++++++ .../RL/tensorflow/non_parallel/DQN_pr.py | 58 +++++++++++ 7.0/models/RL/tensorflow/non_parallel/PPO.py | 68 +++++++++++++ .../RL/tensorflow/non_parallel/PPO_eps.py | 63 ++++++++++++ 7.0/models/RL/tensorflow/parallel/DDPG.py | 77 +++++++++++++++ 7.0/models/RL/tensorflow/parallel/DQN.py | 51 ++++++++++ 7.0/models/RL/tensorflow/parallel/PPO.py | 68 +++++++++++++ 7.0/models/RL/tensorflow/parallel/PPO_eps.py | 64 +++++++++++++ 30 files changed, 1636 insertions(+) create mode 100644 7.0/models/DL/pytorch/non_parallel/nn.py create mode 100644 7.0/models/DL/pytorch/parallel/nn.py create mode 100644 7.0/models/DL/pytorch/parallel/nn_device.py create mode 100644 7.0/models/DL/tensorflow/non_parallel/cnn.py create mode 100644 7.0/models/DL/tensorflow/non_parallel/layer/cnn.py create mode 100644 7.0/models/DL/tensorflow/non_parallel/layer/nn.py create mode 100644 7.0/models/DL/tensorflow/non_parallel/nn.py create mode 100644 7.0/models/DL/tensorflow/non_parallel/nn_acc.py create mode 100644 7.0/models/DL/tensorflow/non_parallel/nn_ol.py create mode 100644 7.0/models/DL/tensorflow/parallel/cnn.py create mode 100644 7.0/models/DL/tensorflow/parallel/nn.py create mode 100644 7.0/models/DL/tensorflow/parallel/nn_acc.py create mode 100644 7.0/models/DL/tensorflow/parallel/nn_attenuate.py create mode 100644 7.0/models/DL/tensorflow/parallel/nn_clipping.py create mode 100644 7.0/models/DL/tensorflow/parallel/nn_device.py create mode 100644 7.0/models/RL/pytorch/non_parallel/DDPG.py create mode 100644 7.0/models/RL/pytorch/non_parallel/DQN.py create mode 100644 7.0/models/RL/pytorch/non_parallel/DQN_pr.py create mode 100644 7.0/models/RL/pytorch/non_parallel/DoubleDQN.py create mode 100644 7.0/models/RL/pytorch/non_parallel/DuelingDQN.py create mode 100644 7.0/models/RL/pytorch/parallel/DQN.py create mode 100644 7.0/models/RL/tensorflow/non_parallel/DDPG.py create mode 100644 7.0/models/RL/tensorflow/non_parallel/DQN.py create mode 100644 7.0/models/RL/tensorflow/non_parallel/DQN_pr.py create mode 100644 7.0/models/RL/tensorflow/non_parallel/PPO.py create mode 100644 7.0/models/RL/tensorflow/non_parallel/PPO_eps.py create mode 100644 7.0/models/RL/tensorflow/parallel/DDPG.py create mode 100644 7.0/models/RL/tensorflow/parallel/DQN.py create mode 100644 7.0/models/RL/tensorflow/parallel/PPO.py create mode 100644 7.0/models/RL/tensorflow/parallel/PPO_eps.py diff --git a/7.0/models/DL/pytorch/non_parallel/nn.py b/7.0/models/DL/pytorch/non_parallel/nn.py new file mode 100644 index 00000000..1faec0fc --- /dev/null +++ b/7.0/models/DL/pytorch/non_parallel/nn.py @@ -0,0 +1,53 @@ +import torch +from torch import nn + + +class NeuralNetwork(nn.Module): + def __init__(self): + super().__init__() + self.flatten=nn.Flatten() + self.linear_relu_stack=nn.Sequential( + nn.Linear(28*28, 512), + nn.ReLU(), + nn.Linear(512, 512), + nn.ReLU(), + nn.Linear(512, 10) + ) + + + def forward(self,x): + x=self.flatten(x) + logits=self.linear_relu_stack(x) + return logits + + +class neuralnetwork: + def __init__(self): + if torch.cuda.is_available(): + self.device=torch.device('cuda') + else: + self.device=torch.device('cpu') + self.model=NeuralNetwork().to(self.device) + self.loss_fn=nn.CrossEntropyLoss() + self.opt=torch.optim.SGD(self.model.parameters(),lr=1e-3) #optimizer,kernel uses it to optimize. + + + def fp(self,x): #forward propagation function,kernel uses it for forward propagation. + pred=self.model(x.to(self.device)) + return pred + + + def loss(self,output,labels): #loss functino,kernel uses it to calculate loss. + loss=self.loss_fn(output,labels.to(self.device)) + return loss + + + def backward(self,loss): #backward function,kernel uses it for backpropagation. + self.optim.zero_grad() + loss.backward() + return + + + def opt(self): #opt function,kernel uses it to optimize. + self.optim.step() + return \ No newline at end of file diff --git a/7.0/models/DL/pytorch/parallel/nn.py b/7.0/models/DL/pytorch/parallel/nn.py new file mode 100644 index 00000000..d9b05567 --- /dev/null +++ b/7.0/models/DL/pytorch/parallel/nn.py @@ -0,0 +1,41 @@ +import torch # import the PyTorch library +from torch import nn # import the neural network module from PyTorch + +class NeuralNetwork(nn.Module): # define a class for the neural network model + def __init__(self): # define the constructor method + super().__init__() # call the parent class constructor + self.flatten = nn.Flatten() # define a layer to flatten the input + self.linear_relu_stack=nn.Sequential( # define a stack of linear and relu layers + nn.Linear(28*28,512), # a linear layer with 28*28 input features and 512 output features + nn.ReLU(), # a relu activation function + nn.Linear(512,512), # another linear layer with 512 input and output features + nn.ReLU(), # another relu activation function + nn.Linear(512,10) # a final linear layer with 512 input features and 10 output features + ) + + + def forward(self,x): # define the forward method + x = self.flatten(x) # flatten the input x + logits=self.linear_relu_stack(x) # pass x through the linear relu stack and get the logits + return logits # return the logits + + +class neuralnetwork: # define another class for the neural network object + def __init__(self): # define the constructor method + if torch.cuda.is_available(): # check if cuda is available + self.device=torch.device('cuda') # set the device to cuda + else: + self.device=torch.device('cpu') # otherwise set the device to cpu + self.model=NeuralNetwork().to(self.device) # create an instance of the NeuralNetwork class and move it to the device + self.loss_fn=nn.CrossEntropyLoss() # define the loss function as cross entropy loss + self.opt=[torch.optim.SGD(self.model.parameters(),lr=1e-3) for _ in range(7)] # define a list of three stochastic gradient descent optimizers with learning rate 1e-3 + + + def fp(self,x): # define a method for forward propagation + pred=self.model(x.to(self.device)) # get the predictions from the model by passing x to the device + return pred # return the predictions + + + def loss(self,output,labels): # define a method for calculating the loss + loss=self.loss_fn(output,labels.to(self.device)) # compute the loss by passing output and labels to the device + return loss # return the loss \ No newline at end of file diff --git a/7.0/models/DL/pytorch/parallel/nn_device.py b/7.0/models/DL/pytorch/parallel/nn_device.py new file mode 100644 index 00000000..ad4b8069 --- /dev/null +++ b/7.0/models/DL/pytorch/parallel/nn_device.py @@ -0,0 +1,45 @@ +import torch # import the PyTorch library +from torch import nn # import the neural network module from PyTorch +from Note.nn.parallel.assign_device_pytorch import assign_device # import a custom function to assign device for PyTorch tensors + + +class NeuralNetwork(nn.Module): # define a class for the neural network model + def __init__(self): # define the constructor method + super().__init__() # call the parent class constructor + self.flatten = nn.Flatten() # define a layer to flatten the input tensor + self.linear_relu_stack=nn.Sequential( # define a stack of linear and relu layers + nn.Linear(28*28,512), # define a linear layer with input size 28*28 and output size 512 + nn.ReLU(), # define a relu activation function + nn.Linear(512,512), # define another linear layer with input and output size 512 + nn.ReLU(), # define another relu activation function + nn.Linear(512,10) # define the final linear layer with output size 10 + ) + + + def forward(self,x): # define the forward method for the model + x = self.flatten(x) # flatten the input tensor + logits=self.linear_relu_stack(x) # pass the flattened tensor through the linear relu stack + return logits # return the logits as the output + + +class neuralnetwork: # define another class for the neural network object + def __init__(self): # define the constructor method + if torch.cuda.is_available(): # check if cuda device is available + self.device=torch.device('cuda') # set the device to cuda + else: # otherwise + self.device=torch.device('cpu') # set the device to cpu + self.model=NeuralNetwork().to(self.device) # create an instance of the neural network model and move it to the device + self.loss_fn=nn.CrossEntropyLoss() # define the loss function as cross entropy loss + self.opt=[torch.optim.SGD(self.model.parameters(),lr=1e-3) for _ in range(7)] # define a list of optimizers as stochastic gradient descent with learning rate 1e-3 + + + def fp(self,x,p): # define a method for forward propagation + pred=self.model(x.to(assign_device(p,'GPU'))) # assign the device according to the process index p + # get the predictions from the model by passing the input tensor to the device and then to the model + return pred # return the predictions + + + def loss(self,output,labels,p): # define a method for calculating loss + loss=self.loss_fn(output,labels.to(assign_device(p,'GPU'))) # assign the device according to the process index p + # calculate the loss by passing the output and labels tensors to the device and then to the loss function + return loss # return the loss \ No newline at end of file diff --git a/7.0/models/DL/tensorflow/non_parallel/cnn.py b/7.0/models/DL/tensorflow/non_parallel/cnn.py new file mode 100644 index 00000000..bd7ad1bf --- /dev/null +++ b/7.0/models/DL/tensorflow/non_parallel/cnn.py @@ -0,0 +1,27 @@ +import tensorflow as tf +from tensorflow.keras import layers,models + + +class cnn: + def __init__(self): + self.model=models.Sequential() + self.model.add(layers.Conv2D(32,(3,3),activation='relu',input_shape=(32,32,3))) + self.model.add(layers.MaxPooling2D((2,2))) + self.model.add(layers.Conv2D(64,(3,3),activation='relu')) + self.model.add(layers.MaxPooling2D((2,2))) + self.model.add(layers.Conv2D(64,(3,3),activation='relu')) + self.model.add(layers.Flatten()) + self.model.add(layers.Dense(64,activation='relu')) + self.model.add(layers.Dense(10)) + self.param=self.model.weights + self.opt=tf.keras.optimizers.Adam() + + + def fp(self,data): + output=self.model(data) + return output + + + def loss(self,output,labels): + loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + return loss(labels,output) \ No newline at end of file diff --git a/7.0/models/DL/tensorflow/non_parallel/layer/cnn.py b/7.0/models/DL/tensorflow/non_parallel/layer/cnn.py new file mode 100644 index 00000000..cf396557 --- /dev/null +++ b/7.0/models/DL/tensorflow/non_parallel/layer/cnn.py @@ -0,0 +1,48 @@ +import tensorflow as tf +from Note.nn.layer.conv2d import conv2d +from Note.nn.layer.dense import dense +from Note.nn.layer.flatten import flatten +from Note.nn.Model import Model + +""" +This is an example of using the Note layer module. +""" + +# Define a convolutional neural network class +class cnn: + def __init__(self): + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.opt=tf.keras.optimizers.Adam() + + + def build(self): + # Create three convolutional layers with relu activations + self.conv1=conv2d(32,[3,3],strides=(1,1),padding='SAME',activation='relu') + self.conv2=conv2d(64,[3,3],strides=(2,2),padding='SAME',activation='relu') + self.conv3=conv2d(64,[3,3],strides=(1,1),padding='SAME',activation='relu') + self.flatten=flatten() + # Create two dense layers with relu and linear activations + self.dense1=dense(64,activation='relu') + self.dense2=dense(10) + # Store the parameters of the layers in a list + self.param=Model.param + return + + + def fp(self,data): + # Perform forward propagation on the input data + x=self.conv1(data) # First convolutional layer + x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # First max pooling layer + x=self.conv2(x) # Second convolutional layer + x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # Second max pooling layer + x=self.conv3(x) # Third convolutional layer + x=self.flatten(x) # Flatten the output to a vector + x=self.dense1(x) # First dense layer with relu activation + output=self.dense2.output(x) # Output layer with linear activation + return output + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) \ No newline at end of file diff --git a/7.0/models/DL/tensorflow/non_parallel/layer/nn.py b/7.0/models/DL/tensorflow/non_parallel/layer/nn.py new file mode 100644 index 00000000..7dc63176 --- /dev/null +++ b/7.0/models/DL/tensorflow/non_parallel/layer/nn.py @@ -0,0 +1,38 @@ +import tensorflow as tf +from Note.nn.layer.dense import dense +from Note.nn.layer.flatten import flatten +from Note.nn.Model import Model + +""" +This is an example of using the Note layer module. +""" + +# Define a neural network class +class nn: + def __init__(self): + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.opt=tf.keras.optimizers.Adam() #optimizer,kernel uses it to optimize. + + + def build(self): + # Create two dense layers with relu and linear activations + self.layer1=dense(128,activation='relu') + self.layer2=dense(10) + self.flatten=flatten() + # Store the parameters of the layers in a list + self.param=Model.param + return + + + def fp(self,data): + # Perform forward propagation on the input data + data=self.flatten(data) + output1=self.layer1(data) + output2=self.layer2(output1) + return output2 + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) \ No newline at end of file diff --git a/7.0/models/DL/tensorflow/non_parallel/nn.py b/7.0/models/DL/tensorflow/non_parallel/nn.py new file mode 100644 index 00000000..b4f03a92 --- /dev/null +++ b/7.0/models/DL/tensorflow/non_parallel/nn.py @@ -0,0 +1,23 @@ +import tensorflow as tf + + +class nn: + def __init__(self): + self.model=tf.keras.models.Sequential([ + tf.keras.layers.Flatten(input_shape=(28, 28)), + tf.keras.layers.Dense(128,activation='relu'), + tf.keras.layers.Dropout(0.2), + tf.keras.layers.Dense(10) + ]) + self.param=self.model.weights #parameter list,kernel uses it list for backpropagation. + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.opt=tf.keras.optimizers.Adam() #optimizer,kernel uses it to optimize. + + + def fp(self,data): #forward propagation function,kernel uses it for forward propagation. + output=self.model(data) + return output + + + def loss(self,output,labels): #loss functino,kernel uses it to calculate loss. + return self.loss_object(labels,output) \ No newline at end of file diff --git a/7.0/models/DL/tensorflow/non_parallel/nn_acc.py b/7.0/models/DL/tensorflow/non_parallel/nn_acc.py new file mode 100644 index 00000000..0a726603 --- /dev/null +++ b/7.0/models/DL/tensorflow/non_parallel/nn_acc.py @@ -0,0 +1,28 @@ +import tensorflow as tf + +#An example with accuracy function. +class nn: + def __init__(self): + self.model=tf.keras.models.Sequential([ + tf.keras.layers.Flatten(input_shape=(28, 28)), + tf.keras.layers.Dense(128,activation='relu'), + tf.keras.layers.Dropout(0.2), + tf.keras.layers.Dense(10) + ]) + self.param=self.model.weights #parameter list,kernel uses it list for backpropagation. + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + self.train_accuracy=tf.keras.metrics.SparseCategoricalAccuracy(name='train_accuracy') + self.opt=tf.keras.optimizers.Adam() #optimizer,kernel uses it to optimize. + + + def fp(self,data): #forward propagation function,kernel uses it for forward propagation. + output=self.model(data) + return output + + + def loss(self,output,labels): #loss functino,kernel uses it to calculate loss. + return self.loss_object(labels,output) + + + def accuracy(self,output,labels): #accuracy function,kernel uses it to calculate accuracy. + return self.train_accuracy(labels,output) \ No newline at end of file diff --git a/7.0/models/DL/tensorflow/non_parallel/nn_ol.py b/7.0/models/DL/tensorflow/non_parallel/nn_ol.py new file mode 100644 index 00000000..9dfa608d --- /dev/null +++ b/7.0/models/DL/tensorflow/non_parallel/nn_ol.py @@ -0,0 +1,37 @@ +import tensorflow as tf # import TensorFlow library +import numpy as np # import NumPy library + +# This is an example of online training. +class nn: # define a class for the neural network + def __init__(self,data,labels): # initialize the network with data and labels + self.train_data=data # store the training data + self.train_labels=labels # store the training labels + self.model=tf.keras.models.Sequential([ # create a sequential model with four layers + tf.keras.layers.Flatten(input_shape=(28, 28)), # flatten the input to a vector of 28*28 elements + tf.keras.layers.Dense(128,activation='relu'), # add a dense layer with 128 units and ReLU activation + tf.keras.layers.Dropout(0.2), # add a dropout layer with 0.2 dropout rate to prevent overfitting + tf.keras.layers.Dense(10) # add a dense layer with 10 units for the output logits + ]) + self.param=self.model.weights # parameter list, kernel uses it list for backpropagation + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) # loss object, kernel uses it to calculate loss. Here we use sparse categorical crossentropy with logits as output + self.opt=tf.keras.optimizers.Adam() # optimizer, kernel uses it to optimize. Here we use Adam optimizer + self.train_loss_list=[] # a list to store the training loss values + self.counter=0 # counter to keep track of the number of online updates + self.max_length=1000 # maximum length of train_loss_list + + + def online(self): # This is simulative online function, kernel uses it to get online data and labels + index=np.random.choice(60000,size=[32]) # randomly sample 32 indices from 0 to 59999 + if self.counter==10000: # if the counter reaches 10000, stop the online training + return 'stop' + else: # otherwise, return the data and labels corresponding to the sampled indices + return [self.train_data[index],self.train_labels[index]] + + + def fp(self,data): # forward propagation function, kernel uses it for forward propagation + output=self.model(data) # pass the data through the model and get the output logits + return output + + + def loss(self,output,labels): # loss function, kernel uses it to calculate loss + return self.loss_object(labels,output) # return the sparse categorical crossentropy loss between labels and output \ No newline at end of file diff --git a/7.0/models/DL/tensorflow/parallel/cnn.py b/7.0/models/DL/tensorflow/parallel/cnn.py new file mode 100644 index 00000000..6e084ee8 --- /dev/null +++ b/7.0/models/DL/tensorflow/parallel/cnn.py @@ -0,0 +1,51 @@ +import tensorflow as tf +from Note.nn.layer.conv2d import conv2d +from Note.nn.layer.dense import dense +from Note.nn.layer.flatten import flatten +from Note.nn.parallel.optimizer import Adam +from Note.nn.Model import Model + +# Define a convolutional neural network class +class cnn: + def __init__(self): + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + + + def build(self): + # Create three convolutional layers with relu activations + self.conv1=conv2d(32,[3,3],3,strides=(1,1),padding='SAME',activation='relu') + self.conv2=conv2d(64,[3,3],32,strides=(2,2),padding='SAME',activation='relu') + self.conv3=conv2d(64,[3,3],64,strides=(1,1),padding='SAME',activation='relu') + self.flatten=flatten() + # Create two dense layers with relu and linear activations + self.dense1=dense(64,64,activation='relu') + self.dense2=dense(10,64) + self.optimizer=Adam() + # Store the parameters of the layers in a list + self.param=Model.param + return + + + def fp(self,data): + # Perform forward propagation on the input data + x=self.conv1(data) # First convolutional layer + x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # First max pooling layer + x=self.conv2(x) # Second convolutional layer + x=tf.nn.max_pool2d(x,ksize=(2,2),strides=(2,2),padding='VALID') # Second max pooling layer + x=self.conv3(x) # Third convolutional layer + x=self.flatten(x) # Flatten the output to a vector + x=self.dense1(x) # First dense layer with relu activation + output=self.dense2.output(x) # Output layer with linear activation + return output + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) + + + def opt(self,gradient): + # Perform optimization on the parameters using the gradient and the batch count + param=self.optimizer(gradient,self.param,self.bc[0]) + return param \ No newline at end of file diff --git a/7.0/models/DL/tensorflow/parallel/nn.py b/7.0/models/DL/tensorflow/parallel/nn.py new file mode 100644 index 00000000..db171189 --- /dev/null +++ b/7.0/models/DL/tensorflow/parallel/nn.py @@ -0,0 +1,41 @@ +import tensorflow as tf +from Note.nn.layer.dense import dense +from Note.nn.parallel.optimizer import SGD +from Note.nn.layer.flatten import flatten +from Note.nn.Model import Model + +# Define a neural network class +class nn: + def __init__(self): + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + + + def build(self): + # Create two dense layers with relu and linear activations + self.layer1=dense(128,784,activation='relu') + self.layer2=dense(10,128) + self.flatten=flatten() + self.optimizer=SGD() + # Store the parameters of the layers in a list + self.param=Model.param + return + + + def fp(self,data): + # Perform forward propagation on the input data + data=self.flatten(data) + output1=self.layer1(data) + output2=self.layer2(output1) + return output2 + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) + + + def opt(self,gradient): + # Perform optimization on the parameters using the gradient + param=self.optimizer(gradient,self.param) + return param \ No newline at end of file diff --git a/7.0/models/DL/tensorflow/parallel/nn_acc.py b/7.0/models/DL/tensorflow/parallel/nn_acc.py new file mode 100644 index 00000000..cd3c8026 --- /dev/null +++ b/7.0/models/DL/tensorflow/parallel/nn_acc.py @@ -0,0 +1,46 @@ +import tensorflow as tf +from Note.nn.layer.dense import dense +from Note.nn.parallel.optimizer import SGD +from Note.nn.layer.flatten import flatten +from Note.nn.accuracy import sparse_categorical_accuracy +from Note.nn.Model import Model + +# Define a neural network class +class nn: + def __init__(self): + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + + + def build(self): + # Create two dense layers with relu and linear activations + self.layer1=dense(128,784,activation='relu') + self.layer2=dense(10,128) + self.flatten=flatten() + self.optimizer=SGD() + # Store the parameters of the layers in a list + self.param=Model.param + return + + + def fp(self,data): + # Perform forward propagation on the input data + data=self.flatten(data) + output1=self.layer1(data) + output2=self.layer2(output1) + return output2 + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) + + + def accuracy(self,output,labels): #accuracy function,kernel uses it to calculate accuracy. + return sparse_categorical_accuracy(labels,output) + + + def opt(self,gradient): + # Perform optimization on the parameters using the gradient + param=self.optimizer(gradient,self.param) + return param \ No newline at end of file diff --git a/7.0/models/DL/tensorflow/parallel/nn_attenuate.py b/7.0/models/DL/tensorflow/parallel/nn_attenuate.py new file mode 100644 index 00000000..41f77f62 --- /dev/null +++ b/7.0/models/DL/tensorflow/parallel/nn_attenuate.py @@ -0,0 +1,56 @@ +import tensorflow as tf +from tensorflow.python.util import nest +from Note.nn.layer.dense import dense +from Note.nn.parallel.optimizer import SGD +from Note.nn.layer.flatten import flatten +from Note.nn.accuracy import sparse_categorical_accuracy +from Note.nn.Model import Model + +# Define a neural network class with gradient attenuation +class nn: + def __init__(self): + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + + + def build(self): + # Create two dense layers with relu and linear activations + self.layer1=dense(128,784,activation='relu') + self.layer2=dense(10,128) + self.flatten=flatten() + self.optimizer=SGD() + # Store the parameters of the layers in a list + self.param=Model.param + return + + + def fp(self,data): + # Perform forward propagation on the input data + data=self.flatten(data) # Flatten the data to a one-dimensional vector + output1=self.layer1(data) # Apply the first layer to the data and get the output + output2=self.layer2(output1) # Apply the second layer to the output of the first layer and get the final output + return output2 + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) # Use the loss function to calculate the loss + + + def accuracy(self,output,labels): #accuracy function,kernel uses it to calculate accuracy. + return sparse_categorical_accuracy(labels,output) + + + def attenuate(self,gradient,p): #gradient attenuation function,kernel uses it to calculate attenuation coefficient. + # Apply an exponential decay to the gradient based on the optimization counter + ac=0.9**self.opt_counter[0][p] #ac:attenuation coefficient + gradient_flat=nest.flatten(gradient) # Flatten the gradient to a one-dimensional vector + for i in range(len(gradient_flat)): #self.opt_counter:optimization counter + gradient_flat[i]=tf.cast(ac,gradient_flat[i].dtype)*gradient_flat[i] #p:process number + return gradient + + + def opt(self,gradient): + # Perform optimization on the parameters using the gradient + param=self.optimizer(gradient,self.param) # Use the optimizer to update the parameters + return param \ No newline at end of file diff --git a/7.0/models/DL/tensorflow/parallel/nn_clipping.py b/7.0/models/DL/tensorflow/parallel/nn_clipping.py new file mode 100644 index 00000000..4929a49b --- /dev/null +++ b/7.0/models/DL/tensorflow/parallel/nn_clipping.py @@ -0,0 +1,49 @@ +import tensorflow as tf +from Note.nn.layer.dense import dense +from Note.nn.parallel.optimizer import SGD +from Note.nn.layer.flatten import flatten +from Note.nn.gradient_clipping import gradient_clipping +from Note.nn.Model import Model + +# Define a neural network class +class nn: + def __init__(self): + # Initialize the loss function and the optimizer + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + + + def build(self): + # Create two dense layers with relu and linear activations + self.layer1=dense(128,784,activation='relu') + self.layer2=dense(10,128) + self.flatten=flatten() + self.optimizer=SGD() + # Store the parameters of the layers in a list + self.param=Model.param + return + + + def fp(self,data): + # Perform forward propagation on the input data + data=self.flatten(data) + output1=self.layer1(data) + output2=self.layer2(output1) + return output2 + + + def loss(self,output,labels): + # Compute the loss between the output and the labels + return self.loss_object(labels,output) + + + def gradient(self,tape,loss): + # Compute the gradients of the loss with respect to the parameters + grads=tape.gradient(loss,self.param) + # Clip the gradients by value to avoid exploding gradients + return gradient_clipping(grads,'value',1.0) + + + def opt(self,gradient): + # Perform optimization on the parameters using the gradient + param=self.optimizer(gradient,self.param) + return param \ No newline at end of file diff --git a/7.0/models/DL/tensorflow/parallel/nn_device.py b/7.0/models/DL/tensorflow/parallel/nn_device.py new file mode 100644 index 00000000..7935af2e --- /dev/null +++ b/7.0/models/DL/tensorflow/parallel/nn_device.py @@ -0,0 +1,50 @@ +import tensorflow as tf # import TensorFlow library +from Note.nn.layer.dense import dense # import dense layer class +from Note.nn.layer.flatten import flatten # import Note's flatten layer function +from Note.nn.parallel.optimizer import SGD # import Note's momentum optimizer module +from Note.nn.parallel.assign_device import assign_device # import the function to assign device according to the process index and the device type +from Note.nn.Model import Model + + +class nn: # A neural network class example, allocate device for multiple threads + def __init__(self): # initialize the network + self.loss_object=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) # loss object, kernel uses it to calculate loss. Here we use sparse categorical crossentropy with logits as output + + + def build(self): # build function, kernel uses it to create the network layers + # Create two dense layers with relu and linear activations + self.layer1=dense(128,784,activation='relu') # the first layer with 784 input units and 128 output units and ReLU activation + self.layer2=dense(10,128) # the second layer with 128 input units and 10 output units and linear activation + self.flatten=flatten() + self.optimizer=SGD() + # Store the parameters of the layers in a list + self.param=Model.param # parameter list of both layers, kernel uses it list for backpropagation + return + + + def fp(self,data,p): # forward propagation function, kernel uses it for forward propagation + with tf.device(assign_device(p,'GPU')): # assign the device according to the process index p + data=self.flatten(data) # flatten the data to a vector of 784 elements + output1=self.layer1(data) # pass the data through the first layer and get the output + output2=self.layer2(output1) # pass the output of the first layer through the second layer and get the final output logits + return output2 + + + def loss(self,output,labels,p): # loss function, kernel uses it to calculate loss + with tf.device(assign_device(p,'GPU')): # assign the device according to the process index p + return self.loss_object(labels,output) # return the mean softmax cross entropy loss between labels and output + + + def GradientTape(self,data,labels,p): + with tf.device(assign_device(p,'GPU')): + with tf.GradientTape(persistent=True) as tape: + output=self.fp(data,p) + loss=self.loss(output,labels,p) + return tape,output,loss + + + def opt(self,gradient,p): # optimization function, kernel uses it to optimize parameter + # Perform optimization on the parameters using the gradient + with tf.device(assign_device(p,'GPU')): # assign the device according to the process index p + param=self.optimizer(gradient,self.param) # apply the Note's momentum optimizer to update the parameters using the gradient + return param \ No newline at end of file diff --git a/7.0/models/RL/pytorch/non_parallel/DDPG.py b/7.0/models/RL/pytorch/non_parallel/DDPG.py new file mode 100644 index 00000000..93af197f --- /dev/null +++ b/7.0/models/RL/pytorch/non_parallel/DDPG.py @@ -0,0 +1,96 @@ +import torch +import torch.nn.functional as F +import gym + + +class actor(torch.nn.Module): + def __init__(self,state_dim,hidden_dim,action_dim,action_bound): + super(actor,self).__init__() + self.fc1=torch.nn.Linear(state_dim,hidden_dim) + self.fc2=torch.nn.Linear(hidden_dim,action_dim) + self.action_bound=action_bound + + + def forward(self,x): + x=F.relu(self.fc1(x)) + return torch.tanh(self.fc2(x))*self.action_bound + + +class critic(torch.nn.Module): + def __init__(self,state_dim,hidden_dim,action_dim): + super(critic,self).__init__() + self.fc1=torch.nn.Linear(state_dim+action_dim,hidden_dim) + self.fc2=torch.nn.Linear(hidden_dim,1) + + + def forward(self,x,a): + cat=torch.cat([x,a],dim=1) + x=F.relu(self.fc1(cat)) + return self.fc2(x) + + +class DDPG: + def __init__(self,hidden_dim,sigma,gamma,tau,actor_lr,critic_lr): + if torch.cuda.is_available(): + self.device=torch.device('cuda') + else: + self.device=torch.device('cpu') + self.genv=gym.make('Pendulum-v1') + state_dim=self.genv.observation_space.shape[0] + action_dim=self.genv.action_space.shape[0] + action_bound=self.genv.action_space.high[0] + self.actor=actor(state_dim,hidden_dim,action_dim,action_bound).to(self.device) + self.critic=critic(state_dim,hidden_dim,action_dim).to(self.device) + self.target_actor=actor(state_dim,hidden_dim,action_dim,action_bound).to(self.device) + self.target_critic=critic(state_dim,hidden_dim,action_dim).to(self.device) + self.target_actor.load_state_dict(self.actor.state_dict()) + self.target_critic.load_state_dict(self.critic.state_dict()) + self.sigma=sigma + self.gamma=gamma + self.tau=tau + self.actor_opt=torch.optim.Adam(self.actor.parameters(),lr=actor_lr) + self.critic_opt=torch.optim.Adam(self.critic.parameters(),lr=critic_lr) + + + def env(self,a=None,initial=None): + if initial==True: + state=self.genv.reset(seed=0) + return state + else: + next_state,reward,done,_=self.genv.step(a) + return next_state,reward,done + + + def loss(self,s,a,next_s,r,d): + s=torch.tensor(s,dtype=torch.float).to(self.device) + a=torch.tensor(a,dtype=torch.float).view(-1,1).to(self.device) + next_s=torch.tensor(next_s,dtype=torch.float).to(self.device) + r=torch.tensor(r,dtype=torch.float).view(-1,1).to(self.device) + d=torch.tensor(d,dtype=torch.float).view(-1,1).to(self.device) + next_q_value=self.target_critic(next_s,self.target_actor(next_s)) + q_target=r+self.gamma*next_q_value*(1-d) + actor_loss=-torch.mean(self.critic(s,self.actor(s))) + critic_loss=F.mse_loss(self.critic(s,a),q_target) + return [actor_loss,critic_loss] + + + def backward(self,loss): + self.actor_opt.zero_grad() + loss[0].backward() + self.critic_opt.zero_grad() + loss[1].backward() + return + + + def opt(self): + self.actor_opt.step() + self.critic_opt.step() + return + + + def update_param(self): + for target_param,param in zip(self.target_actor.parameters(),self.actor.parameters()): + target_param.data.copy_(target_param.data*(1.0-self.tau)+param.data*self.tau) + for target_param,param in zip(self.target_critic.parameters(),self.critic.parameters()): + target_param.data.copy_(target_param.data*(1.0-self.tau)+param.data*self.tau) + return \ No newline at end of file diff --git a/7.0/models/RL/pytorch/non_parallel/DQN.py b/7.0/models/RL/pytorch/non_parallel/DQN.py new file mode 100644 index 00000000..a0e58689 --- /dev/null +++ b/7.0/models/RL/pytorch/non_parallel/DQN.py @@ -0,0 +1,64 @@ +import torch +import gym +import torch.nn.functional as F + + +class Qnet(torch.nn.Module): + def __init__(self,state_dim,hidden_dim,action_dim): + super(Qnet,self).__init__() + self.fc1=torch.nn.Linear(state_dim,hidden_dim) + self.fc2=torch.nn.Linear(hidden_dim,action_dim) + + + def forward(self,x): + x=F.relu(self.fc1(x)) + return self.fc2(x) + + +class DQN: + def __init__(self,state_dim,hidden_dim,action_dim): + if torch.cuda.is_available(): + self.device=torch.device('cuda') + else: + self.device=torch.device('cpu') + self.nn=Qnet(state_dim,hidden_dim,action_dim).to(self.device) + self.target_q_net=Qnet(state_dim,hidden_dim,action_dim).to(self.device) + self.optimizer=torch.optim.Adam(self.nn.parameters(),lr=2e-3) #optimizer,kernel uses it to optimize. + self.genv=gym.make('CartPole-v0') #create environment + + + def env(self,a=None,initial=None): #environment function,kernel uses it to interact with the environment. + if initial==True: + state=self.genv.reset() + return state + else: + next_state,reward,done,_=self.genv.step(a) + return next_state,reward,done + + + def loss(self,s,a,next_s,r,d): #loss function,kernel uses it to calculate loss. + s=torch.tensor(s,dtype=torch.float).to(self.device) + a=torch.tensor(a,dtype=torch.int64).view(-1,1).to(self.device) + next_s=torch.tensor(next_s,dtype=torch.float).to(self.device) + r=torch.tensor(r,dtype=torch.float).view(-1,1).to(self.device) + d=torch.tensor(d,dtype=torch.float).view(-1,1).to(self.device) + q_value=self.nn(s).gather(1,a) + next_q_value=self.target_q_net(next_s).max(1)[0].view(-1,1) + target=r+0.98*next_q_value*(1-d) + return F.mse_loss(q_value,target) + + + def backward(self,loss): #backward function,kernel uses it for backpropagation. + self.optimizer.zero_grad() + loss.backward() + return + + + def opt(self): #opt function,kernel uses it to optimize. + self.optimizer.step() + return + + + def update_param(self): #update function,kernel uses it to update parameter. + self.target_q_net.load_state_dict(self.nn.state_dict()) + return \ No newline at end of file diff --git a/7.0/models/RL/pytorch/non_parallel/DQN_pr.py b/7.0/models/RL/pytorch/non_parallel/DQN_pr.py new file mode 100644 index 00000000..66c71847 --- /dev/null +++ b/7.0/models/RL/pytorch/non_parallel/DQN_pr.py @@ -0,0 +1,76 @@ +import torch +import gym +import torch.nn.functional as F +import Note.create.RL.rl.prioritized_replay as pr + +#prioritized replay example +class Qnet(torch.nn.Module): + def __init__(self,state_dim,hidden_dim,action_dim): + super(Qnet,self).__init__() + self.fc1=torch.nn.Linear(state_dim,hidden_dim) + self.fc2=torch.nn.Linear(hidden_dim,action_dim) + + + def forward(self,x): + x=F.relu(self.fc1(x)) + return self.fc2(x) + + +class DQN: + def __init__(self,state_dim,hidden_dim,action_dim): + if torch.cuda.is_available(): + self.device=torch.device('cuda') + else: + self.device=torch.device('cpu') + self.nn=Qnet(state_dim,hidden_dim,action_dim).to(self.device) + self.target_q_net=Qnet(state_dim,hidden_dim,action_dim).to(self.device) + self.pr=pr.pr() + self.initial_TD=7 + self._epsilon=0.0007 + self.alpha=0.7 + self.optimizer=torch.optim.Adam(self.nn.parameters(),lr=2e-3) + self.genv=gym.make('CartPole-v0') + + + def env(self,a=None,initial=None): + if initial==True: + state=self.genv.reset(seed=0) + return state + else: + next_state,reward,done,_=self.genv.step(a) + return next_state,reward,done + + + def data_func(self,state_pool,action_pool,next_state_pool,reward_pool,done_pool,batch): + s,a,next_s,r,d=self.pr.sample(state_pool,action_pool,next_state_pool,reward_pool,done_pool,self.epsilon,self.alpha,batch) + return s,a,next_s,r,d + + + def loss(self,s,a,next_s,r,d): + s=torch.tensor(s,dtype=torch.float).to(self.device) + a=torch.tensor(a,dtype=torch.int64).view(-1,1).to(self.device) + next_s=torch.tensor(next_s,dtype=torch.float).to(self.device) + r=torch.tensor(r,dtype=torch.float).view(-1,1).to(self.device) + d=torch.tensor(d,dtype=torch.float).view(-1,1).to(self.device) + q_value=self.nn(s).gather(1,a) + next_q_value=self.target_q_net(next_s).max(1)[0].view(-1,1) + target=r+0.98*next_q_value*(1-d) + TD=target-q_value + self.pr.update_TD(TD) + return torch.mean(TD**2) + + + def backward(self,loss): + self.optimizer.zero_grad() + loss.backward() + return + + + def opt(self): + self.optimizer.step() + return + + + def update_param(self): + self.target_q_net.load_state_dict(self.nn.state_dict()) + return \ No newline at end of file diff --git a/7.0/models/RL/pytorch/non_parallel/DoubleDQN.py b/7.0/models/RL/pytorch/non_parallel/DoubleDQN.py new file mode 100644 index 00000000..31e606e8 --- /dev/null +++ b/7.0/models/RL/pytorch/non_parallel/DoubleDQN.py @@ -0,0 +1,65 @@ +import torch +import gym +import torch.nn.functional as F + + +class Qnet(torch.nn.Module): + def __init__(self,state_dim,hidden_dim,action_dim): + super(Qnet,self).__init__() + self.fc1=torch.nn.Linear(state_dim,hidden_dim) + self.fc2=torch.nn.Linear(hidden_dim,action_dim) + + + def forward(self,x): + x=F.relu(self.fc1(x)) + return self.fc2(x) + + +class DoubleDQN: + def __init__(self,state_dim,hidden_dim,action_dim): + if torch.cuda.is_available(): + self.device=torch.device('cuda') + else: + self.device=torch.device('cpu') + self.nn=Qnet(state_dim,hidden_dim,action_dim).to(self.device) + self.target_q_net=Qnet(state_dim,hidden_dim,action_dim).to(self.device) + self.optimizer=torch.optim.Adam(self.nn.parameters(),lr=2e-3) + self.genv=gym.make('CartPole-v0') + + + def env(self,a=None,initial=None): + if initial==True: + state=self.genv.reset(seed=0) + return state + else: + next_state,reward,done,_=self.genv.step(a) + return next_state,reward,done + + + def loss(self,s,a,next_s,r,d): + s=torch.tensor(s,dtype=torch.float).to(self.device) + a=torch.tensor(a,dtype=torch.int64).view(-1,1).to(self.device) + next_s=torch.tensor(next_s,dtype=torch.float).to(self.device) + r=torch.tensor(r,dtype=torch.float).view(-1,1).to(self.device) + d=torch.tensor(d,dtype=torch.float).view(-1,1).to(self.device) + q_value=self.nn(s).gather(1,a) + max_action=self.nn(next_s).max(1)[1].view(-1,1) + next_q_value=self.target_q_net(next_s).gather(1,max_action) + target=r+0.98*next_q_value*(1-d) + return F.mse_loss(q_value,target) + + + def backward(self,loss): + self.optimizer.zero_grad() + loss.backward() + return + + + def opt(self): + self.optimizer.step() + return + + + def update_param(self): + self.target_q_net.load_state_dict(self.nn.state_dict()) + return \ No newline at end of file diff --git a/7.0/models/RL/pytorch/non_parallel/DuelingDQN.py b/7.0/models/RL/pytorch/non_parallel/DuelingDQN.py new file mode 100644 index 00000000..b81eeb37 --- /dev/null +++ b/7.0/models/RL/pytorch/non_parallel/DuelingDQN.py @@ -0,0 +1,67 @@ +import torch +import gym +import torch.nn.functional as F + + +class VAnet(torch.nn.Module): + def __init__(self,state_dim,hidden_dim,action_dim): + super(VAnet,self).__init__() + self.fc1=torch.nn.Linear(state_dim,hidden_dim) + self.fc_A=torch.nn.Linear(hidden_dim,action_dim) + self.fc_V=torch.nn.Linear(hidden_dim,1) + + + def forward(self,x): + A=self.fc_A(F.relu(self.fc1(x))) + V=self.fc_V(F.relu(self.fc1(x))) + Q=V+A-A.mean(1).view(-1,1) + return Q + + +class DuelingDQN: + def __init__(self,state_dim,hidden_dim,action_dim): + if torch.cuda.is_available(): + self.device=torch.device('cuda') + else: + self.device=torch.device('cpu') + self.nn=VAnet(state_dim,hidden_dim,action_dim).to(self.device) + self.target_q_net=VAnet(state_dim,hidden_dim,action_dim).to(self.device) + self.optimizer=torch.optim.Adam(self.nn.parameters(),lr=2e-3) + self.genv=gym.make('CartPole-v0') + + + def env(self,a=None,initial=None): + if initial==True: + state=self.genv.reset(seed=0) + return state + else: + next_state,reward,done,_=self.genv.step(a) + return next_state,reward,done + + + def loss(self,s,a,next_s,r,d): + s=torch.tensor(s,dtype=torch.float).to(self.device) + a=torch.tensor(a,dtype=torch.int64).view(-1,1).to(self.device) + next_s=torch.tensor(next_s,dtype=torch.float).to(self.device) + r=torch.tensor(r,dtype=torch.float).view(-1,1).to(self.device) + d=torch.tensor(d,dtype=torch.float).view(-1,1).to(self.device) + q_value=self.nn(s).gather(1,a) + next_q_value=self.target_q_net(next_s).max(1)[0].view(-1,1) + target=r+0.98*next_q_value*(1-d) + return F.mse_loss(q_value,target) + + + def backward(self,loss): + self.optimizer.zero_grad() + loss.backward() + return + + + def opt(self): + self.optimizer.step() + return + + + def update_param(self): + self.target_q_net.load_state_dict(self.nn.state_dict()) + return \ No newline at end of file diff --git a/7.0/models/RL/pytorch/parallel/DQN.py b/7.0/models/RL/pytorch/parallel/DQN.py new file mode 100644 index 00000000..f4d8417d --- /dev/null +++ b/7.0/models/RL/pytorch/parallel/DQN.py @@ -0,0 +1,65 @@ +import torch +import gym +import torch.nn.functional as F +from Note.nn.parallel.assign_device_pytorch import assign_device + + +class Qnet(torch.nn.Module): + def __init__(self,state_dim,hidden_dim,action_dim): + super(Qnet,self).__init__() + self.fc1=torch.nn.Linear(state_dim,hidden_dim) + self.fc2=torch.nn.Linear(hidden_dim,action_dim) + + + def forward(self,x): + x=F.relu(self.fc1(x)) + return self.fc2(x) + + +class DQN: + def __init__(self,state_dim,hidden_dim,action_dim): + if torch.cuda.is_available(): + self.device=torch.device('cuda') + else: + self.device=torch.device('cpu') + self.nn=Qnet(state_dim,hidden_dim,action_dim).to(self.device) + self.target_q_net=Qnet(state_dim,hidden_dim,action_dim).to(self.device) + self.optimizer=[torch.optim.Adam(self.nn.parameters(),lr=2e-3) for _ in range(5)] #optimizer,kernel uses it to optimize. + self.genv=[gym.make('CartPole-v0') for _ in range(5)] #create environment + + + def env(self,a=None,p=None,initial=None): #environment function,kernel uses it to interact with the environment. + if initial==True: + state=self.genv[p].reset() + return state + else: + next_state,reward,done,_=self.genv[p].step(a) + return next_state,reward,done + + + def loss(self,s,a,next_s,r,d,p): #loss function,kernel uses it to calculate loss. + s=torch.tensor(s,dtype=torch.float).to(assign_device(p,'GPU')) + a=torch.tensor(a,dtype=torch.int64).view(-1,1).to(assign_device(p,'GPU')) + next_s=torch.tensor(next_s,dtype=torch.float).to(assign_device(p,'GPU')) + r=torch.tensor(r,dtype=torch.float).view(-1,1).to(assign_device(p,'GPU')) + d=torch.tensor(d,dtype=torch.float).view(-1,1).to(assign_device(p,'GPU')) + q_value=self.nn(s).gather(1,a) + next_q_value=self.target_q_net(next_s).max(1)[0].view(-1,1) + target=r+0.98*next_q_value*(1-d) + return F.mse_loss(q_value,target) + + + def backward(self,loss,p): #backward function,kernel uses it for backpropagation. + self.optimizer[p].zero_grad(set_to_none=True) + loss.backward() + return + + + def opt(self,p): #opt function,kernel uses it to optimize. + self.optimizer[p].step() + return + + + def update_param(self): #update function,kernel uses it to update parameter. + self.target_q_net.load_state_dict(self.nn.state_dict()) + return \ No newline at end of file diff --git a/7.0/models/RL/tensorflow/non_parallel/DDPG.py b/7.0/models/RL/tensorflow/non_parallel/DDPG.py new file mode 100644 index 00000000..33ed8017 --- /dev/null +++ b/7.0/models/RL/tensorflow/non_parallel/DDPG.py @@ -0,0 +1,76 @@ +import tensorflow as tf # import TensorFlow library +from Note import nn +import gym # import OpenAI Gym library + + +class actor: # define a class for the actor network + def __init__(self,state_dim,hidden_dim,action_dim,action_bound): # initialize the network with state dimension, hidden dimension, action dimension and action bound + self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') + self.dense2 = nn.dense(action_dim, hidden_dim, activation='tanh') + self.action_bound=action_bound + self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list + + + def fp(self,x): # forward propagation function, kernel uses it for forward propagation + x = self.dense1(x) + return self.dense2(x)*self.action_bound + + +class critic: # define a class for the critic network + def __init__(self,state_dim,hidden_dim,action_dim): # initialize the network with state dimension, hidden dimension and action dimension + self.dense1 = nn.dense(hidden_dim, state_dim+action_dim, activation='relu') + self.dense2 = nn.dense(action_dim, hidden_dim) + self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list + + + def fp(self,x,a): # forward propagation function, kernel uses it for forward propagation + cat=tf.concat([x,a],axis=1) + x=self.dense1(cat) + return self.dense2(x) + + +class DDPG: # define a class for the DDPG agent + def __init__(self,hidden_dim,sigma,gamma,tau,actor_lr,critic_lr): # initialize the agent with hidden dimension, noise scale, discount factor, soft update factor, actor learning rate and critic learning rate + self.genv=gym.make('Pendulum-v1') # create environment + state_dim=self.genv.observation_space.shape[0] # get the state dimension from the environment observation space + action_dim=self.genv.action_space.shape[0] # get the action dimension from the environment action space + action_bound=self.genv.action_space.high[0] # get the action bound from the environment action space high value (assume symmetric) + self.actor=actor(state_dim,hidden_dim,action_dim,action_bound) # create an actor network for the agent + self.critic=critic(state_dim,hidden_dim,action_dim) # create a critic network for the agent + self.target_actor=actor(state_dim,hidden_dim,action_dim,action_bound) # create a target actor network for the agent + self.target_critic=critic(state_dim,hidden_dim,action_dim) # create a target critic network for the agent + self.actor_param=self.actor.param # parameter list of actor network, kernel uses it list for backpropagation + self.critic_param=self.critic.param # parameter list of critic network, kernel uses it list for backpropagation + nn.assign_param(self.target_actor.param,self.actor_param.copy()) # copy the parameters from actor network to target actor network + nn.assign_param(self.target_critic.param,self.critic_param.copy()) # copy the parameters from critic network to target critic network + self.param=[self.actor_param,self.critic_param] # parameter list of both networks, kernel uses it list for backpropagation + self.sigma=sigma # noise scale + self.gamma=gamma # discount factor + self.tau=tau # soft update factor + self.opt=tf.keras.optimizers.Adam() # optimizer, kernel uses it to optimize. Here we use Adam optimizer + + + def env(self,a=None,initial=None): # environment function, kernel uses it to interact with the environment + if initial==True: # if initial is True, reset the environment and return the initial state + state=self.genv.reset(seed=0) + return state + else: # otherwise, take an action and return the next state, reward and done flag + next_state,reward,done,_=self.genv.step(a) + return next_state,reward,done + + + def loss(self,s,a,next_s,r,d): # loss function, kernel uses it to calculate loss + a=tf.expand_dims(a,axis=1) # expand the action vector to match the Q-value matrix shape + next_q_value=self.target_critic.fp(next_s,self.target_actor.fp(next_s)) # get the Q-value for the next state and action from the target critic network + q_target=tf.cast(r,'float32')+self.gamma*next_q_value*(1-tf.cast(d,'float32')) # calculate the target value using Bellman equation with discount factor gamma + actor_loss=-tf.reduce_mean(self.critic.fp(s,self.actor.fp(s))) # calculate the actor loss as the negative mean of the Q-value for the current state and action from the critic network + critic_loss=tf.reduce_mean((self.critic.fp(s,a)-q_target)**2) # calculate the critic loss as the mean squared error between Q-value and target value + return actor_loss+critic_loss # return a list of actor loss and critic loss + + + def update_param(self): # update function, kernel uses it to update parameter + for target_param,param in zip(self.target_actor.param,self.actor.param): # for each pair of parameters in target actor network and actor network + target_param.assign(target_param*(1.0-self.tau)+param*self.tau) # update the target parameter using soft update with factor tau + for target_param,param in zip(self.target_critic.param,self.critic.param): # for each pair of parameters in target critic network and critic network + target_param.assign(target_param*(1.0-self.tau)+param*self.tau) # update the target parameter using soft update with factor tau + return \ No newline at end of file diff --git a/7.0/models/RL/tensorflow/non_parallel/DQN.py b/7.0/models/RL/tensorflow/non_parallel/DQN.py new file mode 100644 index 00000000..d0be2840 --- /dev/null +++ b/7.0/models/RL/tensorflow/non_parallel/DQN.py @@ -0,0 +1,45 @@ +import tensorflow as tf # import TensorFlow library +from Note import nn +import gym # import OpenAI Gym library + + +class Qnet: # define a class for the Q-network + def __init__(self,state_dim,hidden_dim,action_dim): # initialize the network with state dimension, hidden dimension and action dimension + self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') + self.dense2 = nn.dense(action_dim, hidden_dim) + self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list + + def fp(self,x): # forward propagation function, kernel uses it for forward propagation + x = self.dense2(self.dense1(x)) + return x + + +class DQN: # define a class for the DQN agent + def __init__(self,state_dim,hidden_dim,action_dim): # initialize the agent with state dimension, hidden dimension and action dimension + self.nn=Qnet(state_dim,hidden_dim,action_dim) # create a Q-network for the agent + self.target_q_net=Qnet(state_dim,hidden_dim,action_dim) # create a target Q-network for the agent + self.param=self.nn.param # parameter list, kernel uses it list for backpropagation + self.opt=tf.keras.optimizers.Adam() # optimizer, kernel uses it to optimize + self.genv=gym.make('CartPole-v0') # create environment + + + def env(self,a=None,initial=None): # environment function, kernel uses it to interact with the environment + if initial==True: # if initial is True, reset the environment and return the initial state + state=self.genv.reset(seed=0) + return state + else: # otherwise, take an action and return the next state, reward and done flag + next_state,reward,done,_=self.genv.step(a) + return next_state,reward,done + + + def loss(self,s,a,next_s,r,d): # loss function, kernel uses it to calculate loss + a=tf.expand_dims(a,axis=1) # expand the action vector to match the Q-value matrix shape + q_value=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) # get the Q-value for the selected action + next_q_value=tf.reduce_max(self.target_q_net.fp(next_s),axis=1) # get the maximum Q-value for the next state from the target network + target=tf.cast(r,'float32')+0.98*next_q_value*(1-tf.cast(d,'float32')) # calculate the target value using Bellman equation with discount factor 0.98 + return tf.reduce_mean((q_value-target)**2) # return the mean squared error between Q-value and target value + + + def update_param(self): # update function, kernel uses it to update parameter + nn.assign_param(self.target_q_net.param, self.param) # copy the parameters from the Q-network to the target network + return \ No newline at end of file diff --git a/7.0/models/RL/tensorflow/non_parallel/DQN_pr.py b/7.0/models/RL/tensorflow/non_parallel/DQN_pr.py new file mode 100644 index 00000000..e907903a --- /dev/null +++ b/7.0/models/RL/tensorflow/non_parallel/DQN_pr.py @@ -0,0 +1,58 @@ +import tensorflow as tf +from Note import nn +import gym +import Note.RL.rl.prioritized_replay as pr + + +class Qnet: + def __init__(self,state_dim,hidden_dim,action_dim): + self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') + self.dense2 = nn.dense(action_dim, hidden_dim) + self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list + + def fp(self,x): + x = self.dense2(self.dense1(x)) + return x + + +class DQN: + def __init__(self,state_dim,hidden_dim,action_dim): + self.nn=Qnet(state_dim,hidden_dim,action_dim) + self.target_q_net=Qnet(state_dim,hidden_dim,action_dim) + self.param=self.nn.param + self.pr=pr.pr() + self.initial_TD=7 + self.epsilon=0.0007 + self.alpha=0.7 + self.opt=tf.keras.optimizers.Adam() + self.genv=gym.make('CartPole-v0') + + + def env(self,a=None,initial=None): + if initial==True: + state=self.genv.reset(seed=0) + return state + else: + next_state,reward,done,_=self.genv.step(a) + return next_state,reward,done + + + def data_func(self,state_pool,action_pool,next_state_pool,reward_pool,done_pool,batch): + s,a,next_s,r,d=self.pr.sample(state_pool,action_pool,next_state_pool,reward_pool,done_pool,self.epsilon,self.alpha,batch) + return s,a,next_s,r,d + + + def loss(self,s,a,next_s,r,d): + a=tf.expand_dims(a,axis=1) + q_value=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) + next_q_value=tf.reduce_max(self.target_q_net.fp(next_s),axis=1) + target=tf.cast(r,'float32')+0.98*next_q_value*(1-tf.cast(d,'float32')) + target=tf.expand_dims(target,axis=1) + TD=target-q_value + self.pr.update_TD(TD) + return tf.reduce_mean(TD**2) + + + def update_param(self): + nn.assign_param(self.target_q_net.param, self.param) + return \ No newline at end of file diff --git a/7.0/models/RL/tensorflow/non_parallel/PPO.py b/7.0/models/RL/tensorflow/non_parallel/PPO.py new file mode 100644 index 00000000..c0e9b704 --- /dev/null +++ b/7.0/models/RL/tensorflow/non_parallel/PPO.py @@ -0,0 +1,68 @@ +import tensorflow as tf +from Note import nn +import gym + + +class actor: + def __init__(self,state_dim,hidden_dim,action_dim): + self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') + self.dense2 = nn.dense(action_dim, hidden_dim) + self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list + + def fp(self,x): + x=self.dense1(x) + return tf.nn.softmax(self.dense2(x)) + + +class critic: + def __init__(self,state_dim,hidden_dim): + self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') + self.dense2 = nn.dense(1, hidden_dim) + self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list + + def fp(self,x): + x=self.dense1(x) + return self.dense2(x) + + +class PPO: + def __init__(self,state_dim,hidden_dim,action_dim,clip_eps,alpha): + self.actor=actor(state_dim,hidden_dim,action_dim) + self.nn=actor(state_dim,hidden_dim,action_dim) + nn.assign_param(self.nn.param,self.actor.param.copy()) + self.critic=critic(state_dim,hidden_dim) + self.clip_eps=clip_eps + self.alpha=alpha + self.param=[self.actor.param,self.critic.param] + self.opt=tf.keras.optimizers.Adam() + self.genv=gym.make('CartPole-v0') + + + def env(self,a=None,initial=None): + if initial==True: + state=self.genv.reset(seed=0) + return state + else: + next_state,reward,done,_=self.genv.step(a) + return next_state,reward,done + + + def loss(self,s,a,next_s,r,d): + a=tf.expand_dims(a,axis=1) + action_prob=tf.gather(self.actor.fp(s),a,axis=1,batch_dims=1) + action_prob_old=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) + raito=action_prob/action_prob_old + value=self.critic.fp(s) + value_tar=tf.cast(r,'float32')+0.98*self.critic.fp(next_s)*(1-tf.cast(d,'float32')) + TD=value_tar-value + sur1=raito*TD + sur2=tf.clip_by_value(raito,clip_value_min=1-self.clip_eps,clip_value_max=1+self.clip_eps)*TD + clip_loss=-tf.math.minimum(sur1,sur2) + entropy=action_prob*tf.math.log(action_prob+1e-8) + clip_loss=clip_loss-self.alpha*entropy + return tf.reduce_mean(clip_loss)+tf.reduce_mean((TD)**2) + + + def update_param(self): + nn.assign_param(self.nn.param, self.actor.param) + return \ No newline at end of file diff --git a/7.0/models/RL/tensorflow/non_parallel/PPO_eps.py b/7.0/models/RL/tensorflow/non_parallel/PPO_eps.py new file mode 100644 index 00000000..1287521b --- /dev/null +++ b/7.0/models/RL/tensorflow/non_parallel/PPO_eps.py @@ -0,0 +1,63 @@ +import tensorflow as tf +from Note import nn +import gym + + +class actor: + def __init__(self,state_dim,hidden_dim,action_dim): + self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') + self.dense2 = nn.dense(action_dim, hidden_dim) + self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list + + def fp(self,x): + x=self.dense1(x) + return tf.nn.softmax(self.dense2(x)) + + +class critic: + def __init__(self,state_dim,hidden_dim): + self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') + self.dense2 = nn.dense(1, hidden_dim) + self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list + + def fp(self,x): + x=self.dense1(x) + return self.dense2(x) + + +class PPO: + def __init__(self,state_dim,hidden_dim,action_dim,clip_eps): + self.actor=actor(state_dim,hidden_dim,action_dim) + self.nn=actor(state_dim,hidden_dim,action_dim) + nn.assign_param(self.nn.param,self.actor.param.copy()) + self.critic=critic(state_dim,hidden_dim) + self.clip_eps=clip_eps + self.param=[self.actor.param,self.critic.param] + self.opt=tf.keras.optimizers.Adam() + self.genv=gym.make('CartPole-v0') + + + def env(self,a=None,initial=None): + if initial==True: + state=self.genv.reset(seed=0) + return state + else: + next_state,reward,done,_=self.genv.step(a) + return next_state,reward,done + + + def loss(self,s,a,next_s,r,d): + a=tf.expand_dims(a,axis=1) + raito=tf.gather(self.actor.fp(s),a,axis=1,batch_dims=1)/tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) + value=self.critic.fp(s) + value_tar=tf.cast(r,'float32')+0.98*self.critic.fp(next_s)*(1-tf.cast(d,'float32')) + TD=value_tar-value + sur1=raito*TD + sur2=tf.clip_by_value(raito,clip_value_min=1-self.clip_eps,clip_value_max=1+self.clip_eps)*TD + clip_loss=-tf.math.minimum(sur1,sur2) + return tf.reduce_mean(clip_loss)+tf.reduce_mean((TD)**2) + + + def update_param(self): + nn.assign_param(self.nn.param, self.actor.param) + return \ No newline at end of file diff --git a/7.0/models/RL/tensorflow/parallel/DDPG.py b/7.0/models/RL/tensorflow/parallel/DDPG.py new file mode 100644 index 00000000..ed2959e6 --- /dev/null +++ b/7.0/models/RL/tensorflow/parallel/DDPG.py @@ -0,0 +1,77 @@ +import tensorflow as tf # import TensorFlow library +from Note import nn +import gym # import OpenAI Gym library +import Note.nn.parallel.optimizer as o # import Note's optimizer module + + +class actor: # define a class for the actor network + def __init__(self,state_dim,hidden_dim,action_dim,action_bound): # initialize the network with state dimension, hidden dimension, action dimension and action bound + self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') + self.dense2 = nn.dense(action_dim, hidden_dim, activation='tanh') + self.action_bound=action_bound + self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list + + + def fp(self,x): # forward propagation function, kernel uses it for forward propagation + x = self.dense1(x) + return self.dense2(x)*self.action_bound + + +class critic: # define a class for the critic network + def __init__(self,state_dim,hidden_dim,action_dim): # initialize the network with state dimension, hidden dimension and action dimension + self.dense1 = nn.dense(hidden_dim, state_dim+action_dim, activation='relu') + self.dense2 = nn.dense(action_dim, hidden_dim) + self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list + + + def fp(self,x,a): # forward propagation function, kernel uses it for forward propagation + cat=tf.concat([x,a],axis=1) + x=self.dense1(cat) + return self.dense2(x) + + +class DDPG: # define a class for the DDPG agent + def __init__(self,hidden_dim,sigma,gamma,tau,actor_lr,critic_lr): # initialize the agent with hidden dimension, noise scale, discount factor, soft update factor, actor learning rate and critic learning rate + self.genv=[gym.make('Pendulum-v1') for _ in range(5)] # create a list of 5 environments + state_dim=self.genv.observation_space.shape[0] # get the state dimension from the environment observation space + action_dim=self.genv.action_space.shape[0] # get the action dimension from the environment action space + action_bound=self.genv.action_space.high[0] # get the action bound from the environment action space high value (assume symmetric) + self.actor=actor(state_dim,hidden_dim,action_dim,action_bound) # create an actor network for the agent + self.critic=critic(state_dim,hidden_dim,action_dim) # create a critic network for the agent + self.target_actor=actor(state_dim,hidden_dim,action_dim,action_bound) # create a target actor network for the agent + self.target_critic=critic(state_dim,hidden_dim,action_dim) # create a target critic network for the agent + self.actor_param=self.actor.param # parameter list of actor network, kernel uses it list for backpropagation + self.critic_param=self.critic.param # parameter list of critic network, kernel uses it list for backpropagation + nn.assign_param(self.target_actor.param,self.actor_param.copy()) # copy the parameters from actor network to target actor network + nn.assign_param(self.target_critic.param,self.critic_param.copy()) # copy the parameters from critic network to target critic network + self.param=[self.actor_param,self.critic_param] # parameter list of both networks, kernel uses it list for backpropagation + self.sigma=sigma # noise scale + self.gamma=gamma # discount factor + self.tau=tau # soft update factor + self.optimizer=o.SGD(param=self.param) # optimizer, kernel uses it to optimize. Here we use a custom SGD optimizer + + + def env(self,a=None,p=None,initial=None): # environment function, kernel uses it to interact with the environment + if initial==True: # if initial is True, reset the environment with index p and return the initial state + state=self.genv[p].reset(seed=0) + return state + else: # otherwise, take an action and return the next state, reward and done flag from the environment with index p + next_state,reward,done,_=self.genv[p].step(a) + return next_state,reward,done + + + def loss(self,s,a,next_s,r,d): # loss function, kernel uses it to calculate loss + a=tf.expand_dims(a,axis=1) # expand the action vector to match the Q-value matrix shape + next_q_value=self.target_critic.fp(next_s,self.target_actor.fp(next_s)) # get the Q-value for the next state and action from the target critic network + q_target=tf.cast(r,'float32')+self.gamma*next_q_value*(1-tf.cast(d,'float32')) # calculate the target value using Bellman equation with discount factor gamma + actor_loss=-tf.reduce_mean(self.critic.fp(s,self.actor.fp(s))) # calculate the actor loss as the negative mean of the Q-value for the current state and action from the critic network + critic_loss=tf.reduce_mean((self.critic.fp(s,a)-q_target)**2) # calculate the critic loss as the mean squared error between Q-value and target value + return actor_loss+critic_loss # return a list of actor loss and critic loss + + + def update_param(self): # update function, kernel uses it to update parameter + for target_param,param in zip(self.target_actor.param,self.actor.param): # for each pair of parameters in target actor network and actor network + target_param.assign(target_param*(1.0-self.tau)+param*self.tau) # update the target parameter using soft update with factor tau + for target_param,param in zip(self.target_critic.param,self.critic.param): # for each pair of parameters in target critic network and critic network + target_param.assign(target_param*(1.0-self.tau)+param*self.tau) # update the target parameter using soft update with factor tau + return \ No newline at end of file diff --git a/7.0/models/RL/tensorflow/parallel/DQN.py b/7.0/models/RL/tensorflow/parallel/DQN.py new file mode 100644 index 00000000..9186307d --- /dev/null +++ b/7.0/models/RL/tensorflow/parallel/DQN.py @@ -0,0 +1,51 @@ +import tensorflow as tf # import TensorFlow library +from Note import nn +import gym # import OpenAI Gym library +import Note.nn.parallel.optimizer as o # import Note's optimizer module + + +class Qnet: # define a class for the Q-network + def __init__(self,state_dim,hidden_dim,action_dim): # initialize the network with state dimension, hidden dimension and action dimension + self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') + self.dense2 = nn.dense(action_dim, hidden_dim) + self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list + + def fp(self,x): # forward propagation function, kernel uses it for forward propagation + x = self.dense2(self.dense1(x)) + return x + + +class DQN: # define a class for the DQN agent + def __init__(self,state_dim,hidden_dim,action_dim): # initialize the agent with state dimension, hidden dimension and action dimension + self.nn=Qnet(state_dim,hidden_dim,action_dim) # create a Q-network for the agent + self.target_q_net=Qnet(state_dim,hidden_dim,action_dim) # create a target Q-network for the agent + self.param=self.nn.param # parameter list, kernel uses it list for backpropagation + self.optimizer=o.SGD(param=self.param) # optimizer, kernel uses it to optimize. Here we use a custom SGD optimizer + self.genv=[gym.make('CartPole-v0') for _ in range(5)] # create a list of 5 environments + + + def env(self,a=None,p=None,initial=None): # environment function, kernel uses it to interact with the environment + if initial==True: # if initial is True, reset the environment with index p and return the initial state + state=self.genv[p].reset(seed=0) + return state + else: # otherwise, take an action and return the next state, reward and done flag from the environment with index p + next_state,reward,done,_=self.genv[p].step(a) + return next_state,reward,done + + + def loss(self,s,a,next_s,r,d): # loss function, kernel uses it to calculate loss + a=tf.expand_dims(a,axis=1) # expand the action vector to match the Q-value matrix shape + q_value=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) # get the Q-value for the selected action + next_q_value=tf.reduce_max(self.target_q_net.fp(next_s),axis=1) # get the maximum Q-value for the next state from the target network + target=tf.cast(r,'float32')+0.98*next_q_value*(1-tf.cast(d,'float32')) # calculate the target value using Bellman equation with discount factor 0.98 + return tf.reduce_mean((q_value-target)**2) # return the mean squared error between Q-value and target value + + + def update_param(self): # update function, kernel uses it to update parameter + nn.assign(self.target_q_net.param,self.param) # copy the parameters from the Q-network to the target network + return + + + def opt(self,gradient): # optimization function, kernel uses it to optimize parameter + param=self.optimizer.opt(gradient,self.param) # apply the custom momentum optimizer to update the parameters using the gradient + return param \ No newline at end of file diff --git a/7.0/models/RL/tensorflow/parallel/PPO.py b/7.0/models/RL/tensorflow/parallel/PPO.py new file mode 100644 index 00000000..88763150 --- /dev/null +++ b/7.0/models/RL/tensorflow/parallel/PPO.py @@ -0,0 +1,68 @@ +import tensorflow as tf +from Note import nn +import gym +from Note.nn.parallel.optimizer import SGD + + +class actor: + def __init__(self,state_dim,hidden_dim,action_dim): + self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') + self.dense2 = nn.dense(action_dim, hidden_dim) + self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list + + def fp(self,x): + x=self.dense1(x) + return tf.nn.softmax(self.dense2(x)) + + +class critic: + def __init__(self,state_dim,hidden_dim): + self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') + self.dense2 = nn.dense(1, hidden_dim) + self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list + + def fp(self,x): + x=self.dense1(x) + return self.dense2(x) + + +class PPO: + def __init__(self,state_dim,hidden_dim,action_dim,clip_eps,alpha): + self.actor=actor(state_dim,hidden_dim,action_dim) + self.nn=actor(state_dim,hidden_dim,action_dim) + nn.assign(self.nn.param,self.actor.param.copy()) + self.critic=critic(state_dim,hidden_dim) + self.clip_eps=clip_eps + self.param=[self.actor.param,self.critic.param] + self.opt=SGD(param=self.param) + self.genv=[gym.make('CartPole-v0') for _ in range(5)] + + + def env(self,a=None,p=None,initial=None): + if initial==True: + state=self.genv[p].reset(seed=0) + return state + else: + next_state,reward,done,_=self.genv[p].step(a) + return next_state,reward,done + + + def loss(self,s,a,next_s,r,d): + a=tf.expand_dims(a,axis=1) + action_prob=tf.gather(self.actor.fp(s),a,axis=1,batch_dims=1) + action_prob_old=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) + raito=action_prob/action_prob_old + value=self.critic.fp(s) + value_tar=tf.cast(r,'float32')+0.98*self.critic.fp(next_s)*(1-tf.cast(d,'float32')) + TD=value_tar-value + sur1=raito*TD + sur2=tf.clip_by_value(raito,clip_value_min=1-self.clip_eps,clip_value_max=1+self.clip_eps)*TD + clip_loss=-tf.math.minimum(sur1,sur2) + entropy=action_prob*tf.math.log(action_prob+1e-8) + clip_loss=clip_loss-self.alpha*entropy + return tf.reduce_mean(clip_loss)+tf.reduce_mean((TD)**2) + + + def update_param(self): + nn.assign(self.nn.param,self.actor.param) + return \ No newline at end of file diff --git a/7.0/models/RL/tensorflow/parallel/PPO_eps.py b/7.0/models/RL/tensorflow/parallel/PPO_eps.py new file mode 100644 index 00000000..008baf04 --- /dev/null +++ b/7.0/models/RL/tensorflow/parallel/PPO_eps.py @@ -0,0 +1,64 @@ +import tensorflow as tf +from Note import nn +import gym +from Note.nn.parallel.optimizer import SGD + + +class actor: + def __init__(self,state_dim,hidden_dim,action_dim): + self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') + self.dense2 = nn.dense(action_dim, hidden_dim) + self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list + + def fp(self,x): + x=self.dense1(x) + return tf.nn.softmax(self.dense2(x)) + + +class critic: + def __init__(self,state_dim,hidden_dim): + self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') + self.dense2 = nn.dense(1, hidden_dim) + self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list + + def fp(self,x): + x=self.dense1(x) + return self.dense2(x) + + +class PPO: + def __init__(self,state_dim,hidden_dim,action_dim,clip_eps): + self.actor=actor(state_dim,hidden_dim,action_dim) + self.nn=actor(state_dim,hidden_dim,action_dim) + nn.assign(self.nn.param,self.actor.param.copy()) + self.critic=critic(state_dim,hidden_dim) + self.clip_eps=clip_eps + self.param=[self.actor.param,self.critic.param] + self.opt=SGD(param=self.param) + self.genv=[gym.make('CartPole-v0') for _ in range(5)] + + + def env(self,a=None,p=None,initial=None): + if initial==True: + state=self.genv[p].reset(seed=0) + return state + else: + next_state,reward,done,_=self.genv[p].step(a) + return next_state,reward,done + + + def loss(self,s,a,next_s,r,d): + a=tf.expand_dims(a,axis=1) + raito=tf.gather(self.actor.fp(s),a,axis=1,batch_dims=1)/tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) + value=self.critic.fp(s) + value_tar=tf.cast(r,'float32')+0.98*self.critic.fp(next_s)*(1-tf.cast(d,'float32')) + TD=value_tar-value + sur1=raito*TD + sur2=tf.clip_by_value(raito,clip_value_min=1-self.clip_eps,clip_value_max=1+self.clip_eps)*TD + clip_loss=-tf.math.minimum(sur1,sur2) + return tf.reduce_mean(clip_loss)+tf.reduce_mean((TD)**2) + + + def update_param(self): + nn.assign(self.nn.param,self.actor.param) + return \ No newline at end of file From 8d6153dfb7aa387932290395b12de1de7c6c5714 Mon Sep 17 00:00:00 2001 From: NoteDance Date: Sat, 10 Aug 2024 17:13:48 +0800 Subject: [PATCH 317/337] Update DDPG.py --- 7.0/models/RL/tensorflow/non_parallel/DDPG.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/7.0/models/RL/tensorflow/non_parallel/DDPG.py b/7.0/models/RL/tensorflow/non_parallel/DDPG.py index 33ed8017..ee6ed007 100644 --- a/7.0/models/RL/tensorflow/non_parallel/DDPG.py +++ b/7.0/models/RL/tensorflow/non_parallel/DDPG.py @@ -41,8 +41,8 @@ def __init__(self,hidden_dim,sigma,gamma,tau,actor_lr,critic_lr): # initialize t self.target_critic=critic(state_dim,hidden_dim,action_dim) # create a target critic network for the agent self.actor_param=self.actor.param # parameter list of actor network, kernel uses it list for backpropagation self.critic_param=self.critic.param # parameter list of critic network, kernel uses it list for backpropagation - nn.assign_param(self.target_actor.param,self.actor_param.copy()) # copy the parameters from actor network to target actor network - nn.assign_param(self.target_critic.param,self.critic_param.copy()) # copy the parameters from critic network to target critic network + nn.assign_param(self.target_actor.param,self.actor.param) # copy the parameters from actor network to target actor network + nn.assign_param(self.target_critic.param,self.critic.param) # copy the parameters from critic network to target critic network self.param=[self.actor_param,self.critic_param] # parameter list of both networks, kernel uses it list for backpropagation self.sigma=sigma # noise scale self.gamma=gamma # discount factor @@ -73,4 +73,4 @@ def update_param(self): # update function, kernel uses it to update parameter target_param.assign(target_param*(1.0-self.tau)+param*self.tau) # update the target parameter using soft update with factor tau for target_param,param in zip(self.target_critic.param,self.critic.param): # for each pair of parameters in target critic network and critic network target_param.assign(target_param*(1.0-self.tau)+param*self.tau) # update the target parameter using soft update with factor tau - return \ No newline at end of file + return From 584b90252cd7f844335d0f2f85bfd836a283049c Mon Sep 17 00:00:00 2001 From: NoteDance Date: Sat, 10 Aug 2024 17:14:03 +0800 Subject: [PATCH 318/337] Update DDPG.py --- 7.0/models/RL/tensorflow/parallel/DDPG.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/7.0/models/RL/tensorflow/parallel/DDPG.py b/7.0/models/RL/tensorflow/parallel/DDPG.py index ed2959e6..8209ad26 100644 --- a/7.0/models/RL/tensorflow/parallel/DDPG.py +++ b/7.0/models/RL/tensorflow/parallel/DDPG.py @@ -42,8 +42,8 @@ def __init__(self,hidden_dim,sigma,gamma,tau,actor_lr,critic_lr): # initialize t self.target_critic=critic(state_dim,hidden_dim,action_dim) # create a target critic network for the agent self.actor_param=self.actor.param # parameter list of actor network, kernel uses it list for backpropagation self.critic_param=self.critic.param # parameter list of critic network, kernel uses it list for backpropagation - nn.assign_param(self.target_actor.param,self.actor_param.copy()) # copy the parameters from actor network to target actor network - nn.assign_param(self.target_critic.param,self.critic_param.copy()) # copy the parameters from critic network to target critic network + nn.assign_param(self.target_actor.param,self.actor.param) # copy the parameters from actor network to target actor network + nn.assign_param(self.target_critic.param,self.critic.param) # copy the parameters from critic network to target critic network self.param=[self.actor_param,self.critic_param] # parameter list of both networks, kernel uses it list for backpropagation self.sigma=sigma # noise scale self.gamma=gamma # discount factor @@ -74,4 +74,4 @@ def update_param(self): # update function, kernel uses it to update parameter target_param.assign(target_param*(1.0-self.tau)+param*self.tau) # update the target parameter using soft update with factor tau for target_param,param in zip(self.target_critic.param,self.critic.param): # for each pair of parameters in target critic network and critic network target_param.assign(target_param*(1.0-self.tau)+param*self.tau) # update the target parameter using soft update with factor tau - return \ No newline at end of file + return From 630422f0a0373de855ed7c4860a247c4be662927 Mon Sep 17 00:00:00 2001 From: NoteDance Date: Sun, 11 Aug 2024 14:04:00 +0800 Subject: [PATCH 319/337] Add files via upload --- 7.0/models/RL/RL/pytorch/non_parallel/DDPG.py | 96 +++++++++++++++++++ 7.0/models/RL/RL/pytorch/non_parallel/DQN.py | 64 +++++++++++++ .../RL/RL/pytorch/non_parallel/DQN_pr.py | 76 +++++++++++++++ .../RL/RL/pytorch/non_parallel/DoubleDQN.py | 65 +++++++++++++ .../RL/RL/pytorch/non_parallel/DuelingDQN.py | 67 +++++++++++++ 7.0/models/RL/RL/pytorch/parallel/DQN.py | 65 +++++++++++++ .../RL/RL/tensorflow/non_parallel/DDPG.py | 76 +++++++++++++++ .../RL/RL/tensorflow/non_parallel/DQN.py | 45 +++++++++ .../RL/RL/tensorflow/non_parallel/DQN_pr.py | 58 +++++++++++ .../RL/RL/tensorflow/non_parallel/PPO.py | 68 +++++++++++++ .../RL/RL/tensorflow/non_parallel/PPO_eps.py | 63 ++++++++++++ 7.0/models/RL/RL/tensorflow/parallel/DDPG.py | 77 +++++++++++++++ 7.0/models/RL/RL/tensorflow/parallel/DQN.py | 51 ++++++++++ 7.0/models/RL/RL/tensorflow/parallel/PPO.py | 68 +++++++++++++ .../RL/RL/tensorflow/parallel/PPO_eps.py | 64 +++++++++++++ 15 files changed, 1003 insertions(+) create mode 100644 7.0/models/RL/RL/pytorch/non_parallel/DDPG.py create mode 100644 7.0/models/RL/RL/pytorch/non_parallel/DQN.py create mode 100644 7.0/models/RL/RL/pytorch/non_parallel/DQN_pr.py create mode 100644 7.0/models/RL/RL/pytorch/non_parallel/DoubleDQN.py create mode 100644 7.0/models/RL/RL/pytorch/non_parallel/DuelingDQN.py create mode 100644 7.0/models/RL/RL/pytorch/parallel/DQN.py create mode 100644 7.0/models/RL/RL/tensorflow/non_parallel/DDPG.py create mode 100644 7.0/models/RL/RL/tensorflow/non_parallel/DQN.py create mode 100644 7.0/models/RL/RL/tensorflow/non_parallel/DQN_pr.py create mode 100644 7.0/models/RL/RL/tensorflow/non_parallel/PPO.py create mode 100644 7.0/models/RL/RL/tensorflow/non_parallel/PPO_eps.py create mode 100644 7.0/models/RL/RL/tensorflow/parallel/DDPG.py create mode 100644 7.0/models/RL/RL/tensorflow/parallel/DQN.py create mode 100644 7.0/models/RL/RL/tensorflow/parallel/PPO.py create mode 100644 7.0/models/RL/RL/tensorflow/parallel/PPO_eps.py diff --git a/7.0/models/RL/RL/pytorch/non_parallel/DDPG.py b/7.0/models/RL/RL/pytorch/non_parallel/DDPG.py new file mode 100644 index 00000000..93af197f --- /dev/null +++ b/7.0/models/RL/RL/pytorch/non_parallel/DDPG.py @@ -0,0 +1,96 @@ +import torch +import torch.nn.functional as F +import gym + + +class actor(torch.nn.Module): + def __init__(self,state_dim,hidden_dim,action_dim,action_bound): + super(actor,self).__init__() + self.fc1=torch.nn.Linear(state_dim,hidden_dim) + self.fc2=torch.nn.Linear(hidden_dim,action_dim) + self.action_bound=action_bound + + + def forward(self,x): + x=F.relu(self.fc1(x)) + return torch.tanh(self.fc2(x))*self.action_bound + + +class critic(torch.nn.Module): + def __init__(self,state_dim,hidden_dim,action_dim): + super(critic,self).__init__() + self.fc1=torch.nn.Linear(state_dim+action_dim,hidden_dim) + self.fc2=torch.nn.Linear(hidden_dim,1) + + + def forward(self,x,a): + cat=torch.cat([x,a],dim=1) + x=F.relu(self.fc1(cat)) + return self.fc2(x) + + +class DDPG: + def __init__(self,hidden_dim,sigma,gamma,tau,actor_lr,critic_lr): + if torch.cuda.is_available(): + self.device=torch.device('cuda') + else: + self.device=torch.device('cpu') + self.genv=gym.make('Pendulum-v1') + state_dim=self.genv.observation_space.shape[0] + action_dim=self.genv.action_space.shape[0] + action_bound=self.genv.action_space.high[0] + self.actor=actor(state_dim,hidden_dim,action_dim,action_bound).to(self.device) + self.critic=critic(state_dim,hidden_dim,action_dim).to(self.device) + self.target_actor=actor(state_dim,hidden_dim,action_dim,action_bound).to(self.device) + self.target_critic=critic(state_dim,hidden_dim,action_dim).to(self.device) + self.target_actor.load_state_dict(self.actor.state_dict()) + self.target_critic.load_state_dict(self.critic.state_dict()) + self.sigma=sigma + self.gamma=gamma + self.tau=tau + self.actor_opt=torch.optim.Adam(self.actor.parameters(),lr=actor_lr) + self.critic_opt=torch.optim.Adam(self.critic.parameters(),lr=critic_lr) + + + def env(self,a=None,initial=None): + if initial==True: + state=self.genv.reset(seed=0) + return state + else: + next_state,reward,done,_=self.genv.step(a) + return next_state,reward,done + + + def loss(self,s,a,next_s,r,d): + s=torch.tensor(s,dtype=torch.float).to(self.device) + a=torch.tensor(a,dtype=torch.float).view(-1,1).to(self.device) + next_s=torch.tensor(next_s,dtype=torch.float).to(self.device) + r=torch.tensor(r,dtype=torch.float).view(-1,1).to(self.device) + d=torch.tensor(d,dtype=torch.float).view(-1,1).to(self.device) + next_q_value=self.target_critic(next_s,self.target_actor(next_s)) + q_target=r+self.gamma*next_q_value*(1-d) + actor_loss=-torch.mean(self.critic(s,self.actor(s))) + critic_loss=F.mse_loss(self.critic(s,a),q_target) + return [actor_loss,critic_loss] + + + def backward(self,loss): + self.actor_opt.zero_grad() + loss[0].backward() + self.critic_opt.zero_grad() + loss[1].backward() + return + + + def opt(self): + self.actor_opt.step() + self.critic_opt.step() + return + + + def update_param(self): + for target_param,param in zip(self.target_actor.parameters(),self.actor.parameters()): + target_param.data.copy_(target_param.data*(1.0-self.tau)+param.data*self.tau) + for target_param,param in zip(self.target_critic.parameters(),self.critic.parameters()): + target_param.data.copy_(target_param.data*(1.0-self.tau)+param.data*self.tau) + return \ No newline at end of file diff --git a/7.0/models/RL/RL/pytorch/non_parallel/DQN.py b/7.0/models/RL/RL/pytorch/non_parallel/DQN.py new file mode 100644 index 00000000..a0e58689 --- /dev/null +++ b/7.0/models/RL/RL/pytorch/non_parallel/DQN.py @@ -0,0 +1,64 @@ +import torch +import gym +import torch.nn.functional as F + + +class Qnet(torch.nn.Module): + def __init__(self,state_dim,hidden_dim,action_dim): + super(Qnet,self).__init__() + self.fc1=torch.nn.Linear(state_dim,hidden_dim) + self.fc2=torch.nn.Linear(hidden_dim,action_dim) + + + def forward(self,x): + x=F.relu(self.fc1(x)) + return self.fc2(x) + + +class DQN: + def __init__(self,state_dim,hidden_dim,action_dim): + if torch.cuda.is_available(): + self.device=torch.device('cuda') + else: + self.device=torch.device('cpu') + self.nn=Qnet(state_dim,hidden_dim,action_dim).to(self.device) + self.target_q_net=Qnet(state_dim,hidden_dim,action_dim).to(self.device) + self.optimizer=torch.optim.Adam(self.nn.parameters(),lr=2e-3) #optimizer,kernel uses it to optimize. + self.genv=gym.make('CartPole-v0') #create environment + + + def env(self,a=None,initial=None): #environment function,kernel uses it to interact with the environment. + if initial==True: + state=self.genv.reset() + return state + else: + next_state,reward,done,_=self.genv.step(a) + return next_state,reward,done + + + def loss(self,s,a,next_s,r,d): #loss function,kernel uses it to calculate loss. + s=torch.tensor(s,dtype=torch.float).to(self.device) + a=torch.tensor(a,dtype=torch.int64).view(-1,1).to(self.device) + next_s=torch.tensor(next_s,dtype=torch.float).to(self.device) + r=torch.tensor(r,dtype=torch.float).view(-1,1).to(self.device) + d=torch.tensor(d,dtype=torch.float).view(-1,1).to(self.device) + q_value=self.nn(s).gather(1,a) + next_q_value=self.target_q_net(next_s).max(1)[0].view(-1,1) + target=r+0.98*next_q_value*(1-d) + return F.mse_loss(q_value,target) + + + def backward(self,loss): #backward function,kernel uses it for backpropagation. + self.optimizer.zero_grad() + loss.backward() + return + + + def opt(self): #opt function,kernel uses it to optimize. + self.optimizer.step() + return + + + def update_param(self): #update function,kernel uses it to update parameter. + self.target_q_net.load_state_dict(self.nn.state_dict()) + return \ No newline at end of file diff --git a/7.0/models/RL/RL/pytorch/non_parallel/DQN_pr.py b/7.0/models/RL/RL/pytorch/non_parallel/DQN_pr.py new file mode 100644 index 00000000..66c71847 --- /dev/null +++ b/7.0/models/RL/RL/pytorch/non_parallel/DQN_pr.py @@ -0,0 +1,76 @@ +import torch +import gym +import torch.nn.functional as F +import Note.create.RL.rl.prioritized_replay as pr + +#prioritized replay example +class Qnet(torch.nn.Module): + def __init__(self,state_dim,hidden_dim,action_dim): + super(Qnet,self).__init__() + self.fc1=torch.nn.Linear(state_dim,hidden_dim) + self.fc2=torch.nn.Linear(hidden_dim,action_dim) + + + def forward(self,x): + x=F.relu(self.fc1(x)) + return self.fc2(x) + + +class DQN: + def __init__(self,state_dim,hidden_dim,action_dim): + if torch.cuda.is_available(): + self.device=torch.device('cuda') + else: + self.device=torch.device('cpu') + self.nn=Qnet(state_dim,hidden_dim,action_dim).to(self.device) + self.target_q_net=Qnet(state_dim,hidden_dim,action_dim).to(self.device) + self.pr=pr.pr() + self.initial_TD=7 + self._epsilon=0.0007 + self.alpha=0.7 + self.optimizer=torch.optim.Adam(self.nn.parameters(),lr=2e-3) + self.genv=gym.make('CartPole-v0') + + + def env(self,a=None,initial=None): + if initial==True: + state=self.genv.reset(seed=0) + return state + else: + next_state,reward,done,_=self.genv.step(a) + return next_state,reward,done + + + def data_func(self,state_pool,action_pool,next_state_pool,reward_pool,done_pool,batch): + s,a,next_s,r,d=self.pr.sample(state_pool,action_pool,next_state_pool,reward_pool,done_pool,self.epsilon,self.alpha,batch) + return s,a,next_s,r,d + + + def loss(self,s,a,next_s,r,d): + s=torch.tensor(s,dtype=torch.float).to(self.device) + a=torch.tensor(a,dtype=torch.int64).view(-1,1).to(self.device) + next_s=torch.tensor(next_s,dtype=torch.float).to(self.device) + r=torch.tensor(r,dtype=torch.float).view(-1,1).to(self.device) + d=torch.tensor(d,dtype=torch.float).view(-1,1).to(self.device) + q_value=self.nn(s).gather(1,a) + next_q_value=self.target_q_net(next_s).max(1)[0].view(-1,1) + target=r+0.98*next_q_value*(1-d) + TD=target-q_value + self.pr.update_TD(TD) + return torch.mean(TD**2) + + + def backward(self,loss): + self.optimizer.zero_grad() + loss.backward() + return + + + def opt(self): + self.optimizer.step() + return + + + def update_param(self): + self.target_q_net.load_state_dict(self.nn.state_dict()) + return \ No newline at end of file diff --git a/7.0/models/RL/RL/pytorch/non_parallel/DoubleDQN.py b/7.0/models/RL/RL/pytorch/non_parallel/DoubleDQN.py new file mode 100644 index 00000000..31e606e8 --- /dev/null +++ b/7.0/models/RL/RL/pytorch/non_parallel/DoubleDQN.py @@ -0,0 +1,65 @@ +import torch +import gym +import torch.nn.functional as F + + +class Qnet(torch.nn.Module): + def __init__(self,state_dim,hidden_dim,action_dim): + super(Qnet,self).__init__() + self.fc1=torch.nn.Linear(state_dim,hidden_dim) + self.fc2=torch.nn.Linear(hidden_dim,action_dim) + + + def forward(self,x): + x=F.relu(self.fc1(x)) + return self.fc2(x) + + +class DoubleDQN: + def __init__(self,state_dim,hidden_dim,action_dim): + if torch.cuda.is_available(): + self.device=torch.device('cuda') + else: + self.device=torch.device('cpu') + self.nn=Qnet(state_dim,hidden_dim,action_dim).to(self.device) + self.target_q_net=Qnet(state_dim,hidden_dim,action_dim).to(self.device) + self.optimizer=torch.optim.Adam(self.nn.parameters(),lr=2e-3) + self.genv=gym.make('CartPole-v0') + + + def env(self,a=None,initial=None): + if initial==True: + state=self.genv.reset(seed=0) + return state + else: + next_state,reward,done,_=self.genv.step(a) + return next_state,reward,done + + + def loss(self,s,a,next_s,r,d): + s=torch.tensor(s,dtype=torch.float).to(self.device) + a=torch.tensor(a,dtype=torch.int64).view(-1,1).to(self.device) + next_s=torch.tensor(next_s,dtype=torch.float).to(self.device) + r=torch.tensor(r,dtype=torch.float).view(-1,1).to(self.device) + d=torch.tensor(d,dtype=torch.float).view(-1,1).to(self.device) + q_value=self.nn(s).gather(1,a) + max_action=self.nn(next_s).max(1)[1].view(-1,1) + next_q_value=self.target_q_net(next_s).gather(1,max_action) + target=r+0.98*next_q_value*(1-d) + return F.mse_loss(q_value,target) + + + def backward(self,loss): + self.optimizer.zero_grad() + loss.backward() + return + + + def opt(self): + self.optimizer.step() + return + + + def update_param(self): + self.target_q_net.load_state_dict(self.nn.state_dict()) + return \ No newline at end of file diff --git a/7.0/models/RL/RL/pytorch/non_parallel/DuelingDQN.py b/7.0/models/RL/RL/pytorch/non_parallel/DuelingDQN.py new file mode 100644 index 00000000..b81eeb37 --- /dev/null +++ b/7.0/models/RL/RL/pytorch/non_parallel/DuelingDQN.py @@ -0,0 +1,67 @@ +import torch +import gym +import torch.nn.functional as F + + +class VAnet(torch.nn.Module): + def __init__(self,state_dim,hidden_dim,action_dim): + super(VAnet,self).__init__() + self.fc1=torch.nn.Linear(state_dim,hidden_dim) + self.fc_A=torch.nn.Linear(hidden_dim,action_dim) + self.fc_V=torch.nn.Linear(hidden_dim,1) + + + def forward(self,x): + A=self.fc_A(F.relu(self.fc1(x))) + V=self.fc_V(F.relu(self.fc1(x))) + Q=V+A-A.mean(1).view(-1,1) + return Q + + +class DuelingDQN: + def __init__(self,state_dim,hidden_dim,action_dim): + if torch.cuda.is_available(): + self.device=torch.device('cuda') + else: + self.device=torch.device('cpu') + self.nn=VAnet(state_dim,hidden_dim,action_dim).to(self.device) + self.target_q_net=VAnet(state_dim,hidden_dim,action_dim).to(self.device) + self.optimizer=torch.optim.Adam(self.nn.parameters(),lr=2e-3) + self.genv=gym.make('CartPole-v0') + + + def env(self,a=None,initial=None): + if initial==True: + state=self.genv.reset(seed=0) + return state + else: + next_state,reward,done,_=self.genv.step(a) + return next_state,reward,done + + + def loss(self,s,a,next_s,r,d): + s=torch.tensor(s,dtype=torch.float).to(self.device) + a=torch.tensor(a,dtype=torch.int64).view(-1,1).to(self.device) + next_s=torch.tensor(next_s,dtype=torch.float).to(self.device) + r=torch.tensor(r,dtype=torch.float).view(-1,1).to(self.device) + d=torch.tensor(d,dtype=torch.float).view(-1,1).to(self.device) + q_value=self.nn(s).gather(1,a) + next_q_value=self.target_q_net(next_s).max(1)[0].view(-1,1) + target=r+0.98*next_q_value*(1-d) + return F.mse_loss(q_value,target) + + + def backward(self,loss): + self.optimizer.zero_grad() + loss.backward() + return + + + def opt(self): + self.optimizer.step() + return + + + def update_param(self): + self.target_q_net.load_state_dict(self.nn.state_dict()) + return \ No newline at end of file diff --git a/7.0/models/RL/RL/pytorch/parallel/DQN.py b/7.0/models/RL/RL/pytorch/parallel/DQN.py new file mode 100644 index 00000000..f4d8417d --- /dev/null +++ b/7.0/models/RL/RL/pytorch/parallel/DQN.py @@ -0,0 +1,65 @@ +import torch +import gym +import torch.nn.functional as F +from Note.nn.parallel.assign_device_pytorch import assign_device + + +class Qnet(torch.nn.Module): + def __init__(self,state_dim,hidden_dim,action_dim): + super(Qnet,self).__init__() + self.fc1=torch.nn.Linear(state_dim,hidden_dim) + self.fc2=torch.nn.Linear(hidden_dim,action_dim) + + + def forward(self,x): + x=F.relu(self.fc1(x)) + return self.fc2(x) + + +class DQN: + def __init__(self,state_dim,hidden_dim,action_dim): + if torch.cuda.is_available(): + self.device=torch.device('cuda') + else: + self.device=torch.device('cpu') + self.nn=Qnet(state_dim,hidden_dim,action_dim).to(self.device) + self.target_q_net=Qnet(state_dim,hidden_dim,action_dim).to(self.device) + self.optimizer=[torch.optim.Adam(self.nn.parameters(),lr=2e-3) for _ in range(5)] #optimizer,kernel uses it to optimize. + self.genv=[gym.make('CartPole-v0') for _ in range(5)] #create environment + + + def env(self,a=None,p=None,initial=None): #environment function,kernel uses it to interact with the environment. + if initial==True: + state=self.genv[p].reset() + return state + else: + next_state,reward,done,_=self.genv[p].step(a) + return next_state,reward,done + + + def loss(self,s,a,next_s,r,d,p): #loss function,kernel uses it to calculate loss. + s=torch.tensor(s,dtype=torch.float).to(assign_device(p,'GPU')) + a=torch.tensor(a,dtype=torch.int64).view(-1,1).to(assign_device(p,'GPU')) + next_s=torch.tensor(next_s,dtype=torch.float).to(assign_device(p,'GPU')) + r=torch.tensor(r,dtype=torch.float).view(-1,1).to(assign_device(p,'GPU')) + d=torch.tensor(d,dtype=torch.float).view(-1,1).to(assign_device(p,'GPU')) + q_value=self.nn(s).gather(1,a) + next_q_value=self.target_q_net(next_s).max(1)[0].view(-1,1) + target=r+0.98*next_q_value*(1-d) + return F.mse_loss(q_value,target) + + + def backward(self,loss,p): #backward function,kernel uses it for backpropagation. + self.optimizer[p].zero_grad(set_to_none=True) + loss.backward() + return + + + def opt(self,p): #opt function,kernel uses it to optimize. + self.optimizer[p].step() + return + + + def update_param(self): #update function,kernel uses it to update parameter. + self.target_q_net.load_state_dict(self.nn.state_dict()) + return \ No newline at end of file diff --git a/7.0/models/RL/RL/tensorflow/non_parallel/DDPG.py b/7.0/models/RL/RL/tensorflow/non_parallel/DDPG.py new file mode 100644 index 00000000..6474489a --- /dev/null +++ b/7.0/models/RL/RL/tensorflow/non_parallel/DDPG.py @@ -0,0 +1,76 @@ +import tensorflow as tf # import TensorFlow library +from Note import nn +import gym # import OpenAI Gym library + + +class actor: # define a class for the actor network + def __init__(self,state_dim,hidden_dim,action_dim,action_bound): # initialize the network with state dimension, hidden dimension, action dimension and action bound + self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') + self.dense2 = nn.dense(action_dim, hidden_dim, activation='tanh') + self.action_bound=action_bound + self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list + + + def fp(self,x): # forward propagation function, kernel uses it for forward propagation + x = self.dense1(x) + return self.dense2(x)*self.action_bound + + +class critic: # define a class for the critic network + def __init__(self,state_dim,hidden_dim,action_dim): # initialize the network with state dimension, hidden dimension and action dimension + self.dense1 = nn.dense(hidden_dim, state_dim+action_dim, activation='relu') + self.dense2 = nn.dense(action_dim, hidden_dim) + self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list + + + def fp(self,x,a): # forward propagation function, kernel uses it for forward propagation + cat=tf.concat([x,a],axis=1) + x=self.dense1(cat) + return self.dense2(x) + + +class DDPG: # define a class for the DDPG agent + def __init__(self,hidden_dim,sigma,gamma,tau,actor_lr,critic_lr): # initialize the agent with hidden dimension, noise scale, discount factor, soft update factor, actor learning rate and critic learning rate + self.genv=gym.make('Pendulum-v1') # create environment + state_dim=self.genv.observation_space.shape[0] # get the state dimension from the environment observation space + action_dim=self.genv.action_space.shape[0] # get the action dimension from the environment action space + action_bound=self.genv.action_space.high[0] # get the action bound from the environment action space high value (assume symmetric) + self.actor=actor(state_dim,hidden_dim,action_dim,action_bound) # create an actor network for the agent + self.critic=critic(state_dim,hidden_dim,action_dim) # create a critic network for the agent + self.target_actor=actor(state_dim,hidden_dim,action_dim,action_bound) # create a target actor network for the agent + self.target_critic=critic(state_dim,hidden_dim,action_dim) # create a target critic network for the agent + self.actor_param=self.actor.param # parameter list of actor network, kernel uses it list for backpropagation + self.critic_param=self.critic.param # parameter list of critic network, kernel uses it list for backpropagation + nn.assign_param(self.target_actor.param,self.actor.param) # copy the parameters from actor network to target actor network + nn.assign_param(self.target_critic.param,self.critic.param) # copy the parameters from critic network to target critic network + self.param=[self.actor_param,self.critic_param] # parameter list of both networks, kernel uses it list for backpropagation + self.sigma=sigma # noise scale + self.gamma=gamma # discount factor + self.tau=tau # soft update factor + self.opt=tf.keras.optimizers.Adam() # optimizer, kernel uses it to optimize. Here we use Adam optimizer + + + def env(self,a=None,initial=None): # environment function, kernel uses it to interact with the environment + if initial==True: # if initial is True, reset the environment and return the initial state + state=self.genv.reset(seed=0) + return state + else: # otherwise, take an action and return the next state, reward and done flag + next_state,reward,done,_=self.genv.step(a) + return next_state,reward,done + + + def loss(self,s,a,next_s,r,d): # loss function, kernel uses it to calculate loss + a=tf.expand_dims(a,axis=1) # expand the action vector to match the Q-value matrix shape + next_q_value=self.target_critic.fp(next_s,self.target_actor.fp(next_s)) # get the Q-value for the next state and action from the target critic network + q_target=tf.cast(r,'float32')+self.gamma*next_q_value*(1-tf.cast(d,'float32')) # calculate the target value using Bellman equation with discount factor gamma + actor_loss=-tf.reduce_mean(self.critic.fp(s,self.actor.fp(s))) # calculate the actor loss as the negative mean of the Q-value for the current state and action from the critic network + critic_loss=tf.reduce_mean((self.critic.fp(s,a)-q_target)**2) # calculate the critic loss as the mean squared error between Q-value and target value + return [actor_loss,critic_loss] # return a list of actor loss and critic loss + + + def update_param(self): # update function, kernel uses it to update parameter + for target_param,param in zip(self.target_actor.param,self.actor.param): # for each pair of parameters in target actor network and actor network + target_param.assign(target_param*(1.0-self.tau)+param*self.tau) # update the target parameter using soft update with factor tau + for target_param,param in zip(self.target_critic.param,self.critic.param): # for each pair of parameters in target critic network and critic network + target_param.assign(target_param*(1.0-self.tau)+param*self.tau) # update the target parameter using soft update with factor tau + return \ No newline at end of file diff --git a/7.0/models/RL/RL/tensorflow/non_parallel/DQN.py b/7.0/models/RL/RL/tensorflow/non_parallel/DQN.py new file mode 100644 index 00000000..d0be2840 --- /dev/null +++ b/7.0/models/RL/RL/tensorflow/non_parallel/DQN.py @@ -0,0 +1,45 @@ +import tensorflow as tf # import TensorFlow library +from Note import nn +import gym # import OpenAI Gym library + + +class Qnet: # define a class for the Q-network + def __init__(self,state_dim,hidden_dim,action_dim): # initialize the network with state dimension, hidden dimension and action dimension + self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') + self.dense2 = nn.dense(action_dim, hidden_dim) + self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list + + def fp(self,x): # forward propagation function, kernel uses it for forward propagation + x = self.dense2(self.dense1(x)) + return x + + +class DQN: # define a class for the DQN agent + def __init__(self,state_dim,hidden_dim,action_dim): # initialize the agent with state dimension, hidden dimension and action dimension + self.nn=Qnet(state_dim,hidden_dim,action_dim) # create a Q-network for the agent + self.target_q_net=Qnet(state_dim,hidden_dim,action_dim) # create a target Q-network for the agent + self.param=self.nn.param # parameter list, kernel uses it list for backpropagation + self.opt=tf.keras.optimizers.Adam() # optimizer, kernel uses it to optimize + self.genv=gym.make('CartPole-v0') # create environment + + + def env(self,a=None,initial=None): # environment function, kernel uses it to interact with the environment + if initial==True: # if initial is True, reset the environment and return the initial state + state=self.genv.reset(seed=0) + return state + else: # otherwise, take an action and return the next state, reward and done flag + next_state,reward,done,_=self.genv.step(a) + return next_state,reward,done + + + def loss(self,s,a,next_s,r,d): # loss function, kernel uses it to calculate loss + a=tf.expand_dims(a,axis=1) # expand the action vector to match the Q-value matrix shape + q_value=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) # get the Q-value for the selected action + next_q_value=tf.reduce_max(self.target_q_net.fp(next_s),axis=1) # get the maximum Q-value for the next state from the target network + target=tf.cast(r,'float32')+0.98*next_q_value*(1-tf.cast(d,'float32')) # calculate the target value using Bellman equation with discount factor 0.98 + return tf.reduce_mean((q_value-target)**2) # return the mean squared error between Q-value and target value + + + def update_param(self): # update function, kernel uses it to update parameter + nn.assign_param(self.target_q_net.param, self.param) # copy the parameters from the Q-network to the target network + return \ No newline at end of file diff --git a/7.0/models/RL/RL/tensorflow/non_parallel/DQN_pr.py b/7.0/models/RL/RL/tensorflow/non_parallel/DQN_pr.py new file mode 100644 index 00000000..e907903a --- /dev/null +++ b/7.0/models/RL/RL/tensorflow/non_parallel/DQN_pr.py @@ -0,0 +1,58 @@ +import tensorflow as tf +from Note import nn +import gym +import Note.RL.rl.prioritized_replay as pr + + +class Qnet: + def __init__(self,state_dim,hidden_dim,action_dim): + self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') + self.dense2 = nn.dense(action_dim, hidden_dim) + self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list + + def fp(self,x): + x = self.dense2(self.dense1(x)) + return x + + +class DQN: + def __init__(self,state_dim,hidden_dim,action_dim): + self.nn=Qnet(state_dim,hidden_dim,action_dim) + self.target_q_net=Qnet(state_dim,hidden_dim,action_dim) + self.param=self.nn.param + self.pr=pr.pr() + self.initial_TD=7 + self.epsilon=0.0007 + self.alpha=0.7 + self.opt=tf.keras.optimizers.Adam() + self.genv=gym.make('CartPole-v0') + + + def env(self,a=None,initial=None): + if initial==True: + state=self.genv.reset(seed=0) + return state + else: + next_state,reward,done,_=self.genv.step(a) + return next_state,reward,done + + + def data_func(self,state_pool,action_pool,next_state_pool,reward_pool,done_pool,batch): + s,a,next_s,r,d=self.pr.sample(state_pool,action_pool,next_state_pool,reward_pool,done_pool,self.epsilon,self.alpha,batch) + return s,a,next_s,r,d + + + def loss(self,s,a,next_s,r,d): + a=tf.expand_dims(a,axis=1) + q_value=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) + next_q_value=tf.reduce_max(self.target_q_net.fp(next_s),axis=1) + target=tf.cast(r,'float32')+0.98*next_q_value*(1-tf.cast(d,'float32')) + target=tf.expand_dims(target,axis=1) + TD=target-q_value + self.pr.update_TD(TD) + return tf.reduce_mean(TD**2) + + + def update_param(self): + nn.assign_param(self.target_q_net.param, self.param) + return \ No newline at end of file diff --git a/7.0/models/RL/RL/tensorflow/non_parallel/PPO.py b/7.0/models/RL/RL/tensorflow/non_parallel/PPO.py new file mode 100644 index 00000000..dbce55f2 --- /dev/null +++ b/7.0/models/RL/RL/tensorflow/non_parallel/PPO.py @@ -0,0 +1,68 @@ +import tensorflow as tf +from Note import nn +import gym + + +class actor: + def __init__(self,state_dim,hidden_dim,action_dim): + self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') + self.dense2 = nn.dense(action_dim, hidden_dim) + self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list + + def fp(self,x): + x=self.dense1(x) + return tf.nn.softmax(self.dense2(x)) + + +class critic: + def __init__(self,state_dim,hidden_dim): + self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') + self.dense2 = nn.dense(1, hidden_dim) + self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list + + def fp(self,x): + x=self.dense1(x) + return self.dense2(x) + + +class PPO: + def __init__(self,state_dim,hidden_dim,action_dim,clip_eps,alpha): + self.actor=actor(state_dim,hidden_dim,action_dim) + self.nn=actor(state_dim,hidden_dim,action_dim) + nn.assign_param(self.nn.param,self.actor.param.copy()) + self.critic=critic(state_dim,hidden_dim) + self.clip_eps=clip_eps + self.alpha=alpha + self.param=[self.actor.param,self.critic.param] + self.opt=tf.keras.optimizers.Adam() + self.genv=gym.make('CartPole-v0') + + + def env(self,a=None,initial=None): + if initial==True: + state=self.genv.reset(seed=0) + return state + else: + next_state,reward,done,_=self.genv.step(a) + return next_state,reward,done + + + def loss(self,s,a,next_s,r,d): + a=tf.expand_dims(a,axis=1) + action_prob=tf.gather(self.actor.fp(s),a,axis=1,batch_dims=1) + action_prob_old=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) + raito=action_prob/action_prob_old + value=self.critic.fp(s) + value_tar=tf.cast(r,'float32')+0.98*self.critic.fp(next_s)*(1-tf.cast(d,'float32')) + TD=value_tar-value + sur1=raito*TD + sur2=tf.clip_by_value(raito,clip_value_min=1-self.clip_eps,clip_value_max=1+self.clip_eps)*TD + clip_loss=-tf.math.minimum(sur1,sur2) + entropy=action_prob*tf.math.log(action_prob+1e-8) + clip_loss=clip_loss-self.alpha*entropy + return [tf.reduce_mean(clip_loss),tf.reduce_mean((TD)**2)] + + + def update_param(self): + nn.assign_param(self.nn.param, self.actor.param) + return \ No newline at end of file diff --git a/7.0/models/RL/RL/tensorflow/non_parallel/PPO_eps.py b/7.0/models/RL/RL/tensorflow/non_parallel/PPO_eps.py new file mode 100644 index 00000000..899eccae --- /dev/null +++ b/7.0/models/RL/RL/tensorflow/non_parallel/PPO_eps.py @@ -0,0 +1,63 @@ +import tensorflow as tf +from Note import nn +import gym + + +class actor: + def __init__(self,state_dim,hidden_dim,action_dim): + self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') + self.dense2 = nn.dense(action_dim, hidden_dim) + self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list + + def fp(self,x): + x=self.dense1(x) + return tf.nn.softmax(self.dense2(x)) + + +class critic: + def __init__(self,state_dim,hidden_dim): + self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') + self.dense2 = nn.dense(1, hidden_dim) + self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list + + def fp(self,x): + x=self.dense1(x) + return self.dense2(x) + + +class PPO: + def __init__(self,state_dim,hidden_dim,action_dim,clip_eps): + self.actor=actor(state_dim,hidden_dim,action_dim) + self.nn=actor(state_dim,hidden_dim,action_dim) + nn.assign_param(self.nn.param,self.actor.param.copy()) + self.critic=critic(state_dim,hidden_dim) + self.clip_eps=clip_eps + self.param=[self.actor.param,self.critic.param] + self.opt=tf.keras.optimizers.Adam() + self.genv=gym.make('CartPole-v0') + + + def env(self,a=None,initial=None): + if initial==True: + state=self.genv.reset(seed=0) + return state + else: + next_state,reward,done,_=self.genv.step(a) + return next_state,reward,done + + + def loss(self,s,a,next_s,r,d): + a=tf.expand_dims(a,axis=1) + raito=tf.gather(self.actor.fp(s),a,axis=1,batch_dims=1)/tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) + value=self.critic.fp(s) + value_tar=tf.cast(r,'float32')+0.98*self.critic.fp(next_s)*(1-tf.cast(d,'float32')) + TD=value_tar-value + sur1=raito*TD + sur2=tf.clip_by_value(raito,clip_value_min=1-self.clip_eps,clip_value_max=1+self.clip_eps)*TD + clip_loss=-tf.math.minimum(sur1,sur2) + return [tf.reduce_mean(clip_loss),tf.reduce_mean((TD)**2)] + + + def update_param(self): + nn.assign_param(self.nn.param, self.actor.param) + return \ No newline at end of file diff --git a/7.0/models/RL/RL/tensorflow/parallel/DDPG.py b/7.0/models/RL/RL/tensorflow/parallel/DDPG.py new file mode 100644 index 00000000..2b2f7d99 --- /dev/null +++ b/7.0/models/RL/RL/tensorflow/parallel/DDPG.py @@ -0,0 +1,77 @@ +import tensorflow as tf # import TensorFlow library +from Note import nn +import gym # import OpenAI Gym library +import Note.nn.parallel.optimizer as o # import Note's optimizer module + + +class actor: # define a class for the actor network + def __init__(self,state_dim,hidden_dim,action_dim,action_bound): # initialize the network with state dimension, hidden dimension, action dimension and action bound + self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') + self.dense2 = nn.dense(action_dim, hidden_dim, activation='tanh') + self.action_bound=action_bound + self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list + + + def fp(self,x): # forward propagation function, kernel uses it for forward propagation + x = self.dense1(x) + return self.dense2(x)*self.action_bound + + +class critic: # define a class for the critic network + def __init__(self,state_dim,hidden_dim,action_dim): # initialize the network with state dimension, hidden dimension and action dimension + self.dense1 = nn.dense(hidden_dim, state_dim+action_dim, activation='relu') + self.dense2 = nn.dense(action_dim, hidden_dim) + self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list + + + def fp(self,x,a): # forward propagation function, kernel uses it for forward propagation + cat=tf.concat([x,a],axis=1) + x=self.dense1(cat) + return self.dense2(x) + + +class DDPG: # define a class for the DDPG agent + def __init__(self,hidden_dim,sigma,gamma,tau,actor_lr,critic_lr): # initialize the agent with hidden dimension, noise scale, discount factor, soft update factor, actor learning rate and critic learning rate + self.genv=[gym.make('Pendulum-v1') for _ in range(5)] # create a list of 5 environments + state_dim=self.genv.observation_space.shape[0] # get the state dimension from the environment observation space + action_dim=self.genv.action_space.shape[0] # get the action dimension from the environment action space + action_bound=self.genv.action_space.high[0] # get the action bound from the environment action space high value (assume symmetric) + self.actor=actor(state_dim,hidden_dim,action_dim,action_bound) # create an actor network for the agent + self.critic=critic(state_dim,hidden_dim,action_dim) # create a critic network for the agent + self.target_actor=actor(state_dim,hidden_dim,action_dim,action_bound) # create a target actor network for the agent + self.target_critic=critic(state_dim,hidden_dim,action_dim) # create a target critic network for the agent + self.actor_param=self.actor.param # parameter list of actor network, kernel uses it list for backpropagation + self.critic_param=self.critic.param # parameter list of critic network, kernel uses it list for backpropagation + nn.assign_param(self.target_actor.param,self.actor.param) # copy the parameters from actor network to target actor network + nn.assign_param(self.target_critic.param,self.critic.param) # copy the parameters from critic network to target critic network + self.param=[self.actor_param,self.critic_param] # parameter list of both networks, kernel uses it list for backpropagation + self.sigma=sigma # noise scale + self.gamma=gamma # discount factor + self.tau=tau # soft update factor + self.optimizer=o.SGD(param=self.param) # optimizer, kernel uses it to optimize. Here we use a custom SGD optimizer + + + def env(self,a=None,p=None,initial=None): # environment function, kernel uses it to interact with the environment + if initial==True: # if initial is True, reset the environment with index p and return the initial state + state=self.genv[p].reset(seed=0) + return state + else: # otherwise, take an action and return the next state, reward and done flag from the environment with index p + next_state,reward,done,_=self.genv[p].step(a) + return next_state,reward,done + + + def loss(self,s,a,next_s,r,d): # loss function, kernel uses it to calculate loss + a=tf.expand_dims(a,axis=1) # expand the action vector to match the Q-value matrix shape + next_q_value=self.target_critic.fp(next_s,self.target_actor.fp(next_s)) # get the Q-value for the next state and action from the target critic network + q_target=tf.cast(r,'float32')+self.gamma*next_q_value*(1-tf.cast(d,'float32')) # calculate the target value using Bellman equation with discount factor gamma + actor_loss=-tf.reduce_mean(self.critic.fp(s,self.actor.fp(s))) # calculate the actor loss as the negative mean of the Q-value for the current state and action from the critic network + critic_loss=tf.reduce_mean((self.critic.fp(s,a)-q_target)**2) # calculate the critic loss as the mean squared error between Q-value and target value + return [actor_loss,critic_loss] # return a list of actor loss and critic loss + + + def update_param(self): # update function, kernel uses it to update parameter + for target_param,param in zip(self.target_actor.param,self.actor.param): # for each pair of parameters in target actor network and actor network + target_param.assign(target_param*(1.0-self.tau)+param*self.tau) # update the target parameter using soft update with factor tau + for target_param,param in zip(self.target_critic.param,self.critic.param): # for each pair of parameters in target critic network and critic network + target_param.assign(target_param*(1.0-self.tau)+param*self.tau) # update the target parameter using soft update with factor tau + return \ No newline at end of file diff --git a/7.0/models/RL/RL/tensorflow/parallel/DQN.py b/7.0/models/RL/RL/tensorflow/parallel/DQN.py new file mode 100644 index 00000000..9186307d --- /dev/null +++ b/7.0/models/RL/RL/tensorflow/parallel/DQN.py @@ -0,0 +1,51 @@ +import tensorflow as tf # import TensorFlow library +from Note import nn +import gym # import OpenAI Gym library +import Note.nn.parallel.optimizer as o # import Note's optimizer module + + +class Qnet: # define a class for the Q-network + def __init__(self,state_dim,hidden_dim,action_dim): # initialize the network with state dimension, hidden dimension and action dimension + self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') + self.dense2 = nn.dense(action_dim, hidden_dim) + self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list + + def fp(self,x): # forward propagation function, kernel uses it for forward propagation + x = self.dense2(self.dense1(x)) + return x + + +class DQN: # define a class for the DQN agent + def __init__(self,state_dim,hidden_dim,action_dim): # initialize the agent with state dimension, hidden dimension and action dimension + self.nn=Qnet(state_dim,hidden_dim,action_dim) # create a Q-network for the agent + self.target_q_net=Qnet(state_dim,hidden_dim,action_dim) # create a target Q-network for the agent + self.param=self.nn.param # parameter list, kernel uses it list for backpropagation + self.optimizer=o.SGD(param=self.param) # optimizer, kernel uses it to optimize. Here we use a custom SGD optimizer + self.genv=[gym.make('CartPole-v0') for _ in range(5)] # create a list of 5 environments + + + def env(self,a=None,p=None,initial=None): # environment function, kernel uses it to interact with the environment + if initial==True: # if initial is True, reset the environment with index p and return the initial state + state=self.genv[p].reset(seed=0) + return state + else: # otherwise, take an action and return the next state, reward and done flag from the environment with index p + next_state,reward,done,_=self.genv[p].step(a) + return next_state,reward,done + + + def loss(self,s,a,next_s,r,d): # loss function, kernel uses it to calculate loss + a=tf.expand_dims(a,axis=1) # expand the action vector to match the Q-value matrix shape + q_value=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) # get the Q-value for the selected action + next_q_value=tf.reduce_max(self.target_q_net.fp(next_s),axis=1) # get the maximum Q-value for the next state from the target network + target=tf.cast(r,'float32')+0.98*next_q_value*(1-tf.cast(d,'float32')) # calculate the target value using Bellman equation with discount factor 0.98 + return tf.reduce_mean((q_value-target)**2) # return the mean squared error between Q-value and target value + + + def update_param(self): # update function, kernel uses it to update parameter + nn.assign(self.target_q_net.param,self.param) # copy the parameters from the Q-network to the target network + return + + + def opt(self,gradient): # optimization function, kernel uses it to optimize parameter + param=self.optimizer.opt(gradient,self.param) # apply the custom momentum optimizer to update the parameters using the gradient + return param \ No newline at end of file diff --git a/7.0/models/RL/RL/tensorflow/parallel/PPO.py b/7.0/models/RL/RL/tensorflow/parallel/PPO.py new file mode 100644 index 00000000..dbf28400 --- /dev/null +++ b/7.0/models/RL/RL/tensorflow/parallel/PPO.py @@ -0,0 +1,68 @@ +import tensorflow as tf +from Note import nn +import gym +from Note.nn.parallel.optimizer import SGD + + +class actor: + def __init__(self,state_dim,hidden_dim,action_dim): + self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') + self.dense2 = nn.dense(action_dim, hidden_dim) + self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list + + def fp(self,x): + x=self.dense1(x) + return tf.nn.softmax(self.dense2(x)) + + +class critic: + def __init__(self,state_dim,hidden_dim): + self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') + self.dense2 = nn.dense(1, hidden_dim) + self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list + + def fp(self,x): + x=self.dense1(x) + return self.dense2(x) + + +class PPO: + def __init__(self,state_dim,hidden_dim,action_dim,clip_eps,alpha): + self.actor=actor(state_dim,hidden_dim,action_dim) + self.nn=actor(state_dim,hidden_dim,action_dim) + nn.assign(self.nn.param,self.actor.param.copy()) + self.critic=critic(state_dim,hidden_dim) + self.clip_eps=clip_eps + self.param=[self.actor.param,self.critic.param] + self.opt=SGD(param=self.param) + self.genv=[gym.make('CartPole-v0') for _ in range(5)] + + + def env(self,a=None,p=None,initial=None): + if initial==True: + state=self.genv[p].reset(seed=0) + return state + else: + next_state,reward,done,_=self.genv[p].step(a) + return next_state,reward,done + + + def loss(self,s,a,next_s,r,d): + a=tf.expand_dims(a,axis=1) + action_prob=tf.gather(self.actor.fp(s),a,axis=1,batch_dims=1) + action_prob_old=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) + raito=action_prob/action_prob_old + value=self.critic.fp(s) + value_tar=tf.cast(r,'float32')+0.98*self.critic.fp(next_s)*(1-tf.cast(d,'float32')) + TD=value_tar-value + sur1=raito*TD + sur2=tf.clip_by_value(raito,clip_value_min=1-self.clip_eps,clip_value_max=1+self.clip_eps)*TD + clip_loss=-tf.math.minimum(sur1,sur2) + entropy=action_prob*tf.math.log(action_prob+1e-8) + clip_loss=clip_loss-self.alpha*entropy + return [tf.reduce_mean(clip_loss),tf.reduce_mean((TD)**2)] + + + def update_param(self): + nn.assign(self.nn.param,self.actor.param) + return \ No newline at end of file diff --git a/7.0/models/RL/RL/tensorflow/parallel/PPO_eps.py b/7.0/models/RL/RL/tensorflow/parallel/PPO_eps.py new file mode 100644 index 00000000..4a4eb6af --- /dev/null +++ b/7.0/models/RL/RL/tensorflow/parallel/PPO_eps.py @@ -0,0 +1,64 @@ +import tensorflow as tf +from Note import nn +import gym +from Note.nn.parallel.optimizer import SGD + + +class actor: + def __init__(self,state_dim,hidden_dim,action_dim): + self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') + self.dense2 = nn.dense(action_dim, hidden_dim) + self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list + + def fp(self,x): + x=self.dense1(x) + return tf.nn.softmax(self.dense2(x)) + + +class critic: + def __init__(self,state_dim,hidden_dim): + self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') + self.dense2 = nn.dense(1, hidden_dim) + self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list + + def fp(self,x): + x=self.dense1(x) + return self.dense2(x) + + +class PPO: + def __init__(self,state_dim,hidden_dim,action_dim,clip_eps): + self.actor=actor(state_dim,hidden_dim,action_dim) + self.nn=actor(state_dim,hidden_dim,action_dim) + nn.assign(self.nn.param,self.actor.param.copy()) + self.critic=critic(state_dim,hidden_dim) + self.clip_eps=clip_eps + self.param=[self.actor.param,self.critic.param] + self.opt=SGD(param=self.param) + self.genv=[gym.make('CartPole-v0') for _ in range(5)] + + + def env(self,a=None,p=None,initial=None): + if initial==True: + state=self.genv[p].reset(seed=0) + return state + else: + next_state,reward,done,_=self.genv[p].step(a) + return next_state,reward,done + + + def loss(self,s,a,next_s,r,d): + a=tf.expand_dims(a,axis=1) + raito=tf.gather(self.actor.fp(s),a,axis=1,batch_dims=1)/tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) + value=self.critic.fp(s) + value_tar=tf.cast(r,'float32')+0.98*self.critic.fp(next_s)*(1-tf.cast(d,'float32')) + TD=value_tar-value + sur1=raito*TD + sur2=tf.clip_by_value(raito,clip_value_min=1-self.clip_eps,clip_value_max=1+self.clip_eps)*TD + clip_loss=-tf.math.minimum(sur1,sur2) + return [tf.reduce_mean(clip_loss),tf.reduce_mean((TD)**2)] + + + def update_param(self): + nn.assign(self.nn.param,self.actor.param) + return \ No newline at end of file From 5a919f64b92e2828993ad45b03026ca69a99dd9d Mon Sep 17 00:00:00 2001 From: NoteDance Date: Thu, 19 Sep 2024 15:13:52 +0800 Subject: [PATCH 320/337] Delete 7.0/models/RL/RL directory --- 7.0/models/RL/RL/pytorch/non_parallel/DDPG.py | 96 ------------------- 7.0/models/RL/RL/pytorch/non_parallel/DQN.py | 64 ------------- .../RL/RL/pytorch/non_parallel/DQN_pr.py | 76 --------------- .../RL/RL/pytorch/non_parallel/DoubleDQN.py | 65 ------------- .../RL/RL/pytorch/non_parallel/DuelingDQN.py | 67 ------------- 7.0/models/RL/RL/pytorch/parallel/DQN.py | 65 ------------- .../RL/RL/tensorflow/non_parallel/DDPG.py | 76 --------------- .../RL/RL/tensorflow/non_parallel/DQN.py | 45 --------- .../RL/RL/tensorflow/non_parallel/DQN_pr.py | 58 ----------- .../RL/RL/tensorflow/non_parallel/PPO.py | 68 ------------- .../RL/RL/tensorflow/non_parallel/PPO_eps.py | 63 ------------ 7.0/models/RL/RL/tensorflow/parallel/DDPG.py | 77 --------------- 7.0/models/RL/RL/tensorflow/parallel/DQN.py | 51 ---------- 7.0/models/RL/RL/tensorflow/parallel/PPO.py | 68 ------------- .../RL/RL/tensorflow/parallel/PPO_eps.py | 64 ------------- 15 files changed, 1003 deletions(-) delete mode 100644 7.0/models/RL/RL/pytorch/non_parallel/DDPG.py delete mode 100644 7.0/models/RL/RL/pytorch/non_parallel/DQN.py delete mode 100644 7.0/models/RL/RL/pytorch/non_parallel/DQN_pr.py delete mode 100644 7.0/models/RL/RL/pytorch/non_parallel/DoubleDQN.py delete mode 100644 7.0/models/RL/RL/pytorch/non_parallel/DuelingDQN.py delete mode 100644 7.0/models/RL/RL/pytorch/parallel/DQN.py delete mode 100644 7.0/models/RL/RL/tensorflow/non_parallel/DDPG.py delete mode 100644 7.0/models/RL/RL/tensorflow/non_parallel/DQN.py delete mode 100644 7.0/models/RL/RL/tensorflow/non_parallel/DQN_pr.py delete mode 100644 7.0/models/RL/RL/tensorflow/non_parallel/PPO.py delete mode 100644 7.0/models/RL/RL/tensorflow/non_parallel/PPO_eps.py delete mode 100644 7.0/models/RL/RL/tensorflow/parallel/DDPG.py delete mode 100644 7.0/models/RL/RL/tensorflow/parallel/DQN.py delete mode 100644 7.0/models/RL/RL/tensorflow/parallel/PPO.py delete mode 100644 7.0/models/RL/RL/tensorflow/parallel/PPO_eps.py diff --git a/7.0/models/RL/RL/pytorch/non_parallel/DDPG.py b/7.0/models/RL/RL/pytorch/non_parallel/DDPG.py deleted file mode 100644 index 93af197f..00000000 --- a/7.0/models/RL/RL/pytorch/non_parallel/DDPG.py +++ /dev/null @@ -1,96 +0,0 @@ -import torch -import torch.nn.functional as F -import gym - - -class actor(torch.nn.Module): - def __init__(self,state_dim,hidden_dim,action_dim,action_bound): - super(actor,self).__init__() - self.fc1=torch.nn.Linear(state_dim,hidden_dim) - self.fc2=torch.nn.Linear(hidden_dim,action_dim) - self.action_bound=action_bound - - - def forward(self,x): - x=F.relu(self.fc1(x)) - return torch.tanh(self.fc2(x))*self.action_bound - - -class critic(torch.nn.Module): - def __init__(self,state_dim,hidden_dim,action_dim): - super(critic,self).__init__() - self.fc1=torch.nn.Linear(state_dim+action_dim,hidden_dim) - self.fc2=torch.nn.Linear(hidden_dim,1) - - - def forward(self,x,a): - cat=torch.cat([x,a],dim=1) - x=F.relu(self.fc1(cat)) - return self.fc2(x) - - -class DDPG: - def __init__(self,hidden_dim,sigma,gamma,tau,actor_lr,critic_lr): - if torch.cuda.is_available(): - self.device=torch.device('cuda') - else: - self.device=torch.device('cpu') - self.genv=gym.make('Pendulum-v1') - state_dim=self.genv.observation_space.shape[0] - action_dim=self.genv.action_space.shape[0] - action_bound=self.genv.action_space.high[0] - self.actor=actor(state_dim,hidden_dim,action_dim,action_bound).to(self.device) - self.critic=critic(state_dim,hidden_dim,action_dim).to(self.device) - self.target_actor=actor(state_dim,hidden_dim,action_dim,action_bound).to(self.device) - self.target_critic=critic(state_dim,hidden_dim,action_dim).to(self.device) - self.target_actor.load_state_dict(self.actor.state_dict()) - self.target_critic.load_state_dict(self.critic.state_dict()) - self.sigma=sigma - self.gamma=gamma - self.tau=tau - self.actor_opt=torch.optim.Adam(self.actor.parameters(),lr=actor_lr) - self.critic_opt=torch.optim.Adam(self.critic.parameters(),lr=critic_lr) - - - def env(self,a=None,initial=None): - if initial==True: - state=self.genv.reset(seed=0) - return state - else: - next_state,reward,done,_=self.genv.step(a) - return next_state,reward,done - - - def loss(self,s,a,next_s,r,d): - s=torch.tensor(s,dtype=torch.float).to(self.device) - a=torch.tensor(a,dtype=torch.float).view(-1,1).to(self.device) - next_s=torch.tensor(next_s,dtype=torch.float).to(self.device) - r=torch.tensor(r,dtype=torch.float).view(-1,1).to(self.device) - d=torch.tensor(d,dtype=torch.float).view(-1,1).to(self.device) - next_q_value=self.target_critic(next_s,self.target_actor(next_s)) - q_target=r+self.gamma*next_q_value*(1-d) - actor_loss=-torch.mean(self.critic(s,self.actor(s))) - critic_loss=F.mse_loss(self.critic(s,a),q_target) - return [actor_loss,critic_loss] - - - def backward(self,loss): - self.actor_opt.zero_grad() - loss[0].backward() - self.critic_opt.zero_grad() - loss[1].backward() - return - - - def opt(self): - self.actor_opt.step() - self.critic_opt.step() - return - - - def update_param(self): - for target_param,param in zip(self.target_actor.parameters(),self.actor.parameters()): - target_param.data.copy_(target_param.data*(1.0-self.tau)+param.data*self.tau) - for target_param,param in zip(self.target_critic.parameters(),self.critic.parameters()): - target_param.data.copy_(target_param.data*(1.0-self.tau)+param.data*self.tau) - return \ No newline at end of file diff --git a/7.0/models/RL/RL/pytorch/non_parallel/DQN.py b/7.0/models/RL/RL/pytorch/non_parallel/DQN.py deleted file mode 100644 index a0e58689..00000000 --- a/7.0/models/RL/RL/pytorch/non_parallel/DQN.py +++ /dev/null @@ -1,64 +0,0 @@ -import torch -import gym -import torch.nn.functional as F - - -class Qnet(torch.nn.Module): - def __init__(self,state_dim,hidden_dim,action_dim): - super(Qnet,self).__init__() - self.fc1=torch.nn.Linear(state_dim,hidden_dim) - self.fc2=torch.nn.Linear(hidden_dim,action_dim) - - - def forward(self,x): - x=F.relu(self.fc1(x)) - return self.fc2(x) - - -class DQN: - def __init__(self,state_dim,hidden_dim,action_dim): - if torch.cuda.is_available(): - self.device=torch.device('cuda') - else: - self.device=torch.device('cpu') - self.nn=Qnet(state_dim,hidden_dim,action_dim).to(self.device) - self.target_q_net=Qnet(state_dim,hidden_dim,action_dim).to(self.device) - self.optimizer=torch.optim.Adam(self.nn.parameters(),lr=2e-3) #optimizer,kernel uses it to optimize. - self.genv=gym.make('CartPole-v0') #create environment - - - def env(self,a=None,initial=None): #environment function,kernel uses it to interact with the environment. - if initial==True: - state=self.genv.reset() - return state - else: - next_state,reward,done,_=self.genv.step(a) - return next_state,reward,done - - - def loss(self,s,a,next_s,r,d): #loss function,kernel uses it to calculate loss. - s=torch.tensor(s,dtype=torch.float).to(self.device) - a=torch.tensor(a,dtype=torch.int64).view(-1,1).to(self.device) - next_s=torch.tensor(next_s,dtype=torch.float).to(self.device) - r=torch.tensor(r,dtype=torch.float).view(-1,1).to(self.device) - d=torch.tensor(d,dtype=torch.float).view(-1,1).to(self.device) - q_value=self.nn(s).gather(1,a) - next_q_value=self.target_q_net(next_s).max(1)[0].view(-1,1) - target=r+0.98*next_q_value*(1-d) - return F.mse_loss(q_value,target) - - - def backward(self,loss): #backward function,kernel uses it for backpropagation. - self.optimizer.zero_grad() - loss.backward() - return - - - def opt(self): #opt function,kernel uses it to optimize. - self.optimizer.step() - return - - - def update_param(self): #update function,kernel uses it to update parameter. - self.target_q_net.load_state_dict(self.nn.state_dict()) - return \ No newline at end of file diff --git a/7.0/models/RL/RL/pytorch/non_parallel/DQN_pr.py b/7.0/models/RL/RL/pytorch/non_parallel/DQN_pr.py deleted file mode 100644 index 66c71847..00000000 --- a/7.0/models/RL/RL/pytorch/non_parallel/DQN_pr.py +++ /dev/null @@ -1,76 +0,0 @@ -import torch -import gym -import torch.nn.functional as F -import Note.create.RL.rl.prioritized_replay as pr - -#prioritized replay example -class Qnet(torch.nn.Module): - def __init__(self,state_dim,hidden_dim,action_dim): - super(Qnet,self).__init__() - self.fc1=torch.nn.Linear(state_dim,hidden_dim) - self.fc2=torch.nn.Linear(hidden_dim,action_dim) - - - def forward(self,x): - x=F.relu(self.fc1(x)) - return self.fc2(x) - - -class DQN: - def __init__(self,state_dim,hidden_dim,action_dim): - if torch.cuda.is_available(): - self.device=torch.device('cuda') - else: - self.device=torch.device('cpu') - self.nn=Qnet(state_dim,hidden_dim,action_dim).to(self.device) - self.target_q_net=Qnet(state_dim,hidden_dim,action_dim).to(self.device) - self.pr=pr.pr() - self.initial_TD=7 - self._epsilon=0.0007 - self.alpha=0.7 - self.optimizer=torch.optim.Adam(self.nn.parameters(),lr=2e-3) - self.genv=gym.make('CartPole-v0') - - - def env(self,a=None,initial=None): - if initial==True: - state=self.genv.reset(seed=0) - return state - else: - next_state,reward,done,_=self.genv.step(a) - return next_state,reward,done - - - def data_func(self,state_pool,action_pool,next_state_pool,reward_pool,done_pool,batch): - s,a,next_s,r,d=self.pr.sample(state_pool,action_pool,next_state_pool,reward_pool,done_pool,self.epsilon,self.alpha,batch) - return s,a,next_s,r,d - - - def loss(self,s,a,next_s,r,d): - s=torch.tensor(s,dtype=torch.float).to(self.device) - a=torch.tensor(a,dtype=torch.int64).view(-1,1).to(self.device) - next_s=torch.tensor(next_s,dtype=torch.float).to(self.device) - r=torch.tensor(r,dtype=torch.float).view(-1,1).to(self.device) - d=torch.tensor(d,dtype=torch.float).view(-1,1).to(self.device) - q_value=self.nn(s).gather(1,a) - next_q_value=self.target_q_net(next_s).max(1)[0].view(-1,1) - target=r+0.98*next_q_value*(1-d) - TD=target-q_value - self.pr.update_TD(TD) - return torch.mean(TD**2) - - - def backward(self,loss): - self.optimizer.zero_grad() - loss.backward() - return - - - def opt(self): - self.optimizer.step() - return - - - def update_param(self): - self.target_q_net.load_state_dict(self.nn.state_dict()) - return \ No newline at end of file diff --git a/7.0/models/RL/RL/pytorch/non_parallel/DoubleDQN.py b/7.0/models/RL/RL/pytorch/non_parallel/DoubleDQN.py deleted file mode 100644 index 31e606e8..00000000 --- a/7.0/models/RL/RL/pytorch/non_parallel/DoubleDQN.py +++ /dev/null @@ -1,65 +0,0 @@ -import torch -import gym -import torch.nn.functional as F - - -class Qnet(torch.nn.Module): - def __init__(self,state_dim,hidden_dim,action_dim): - super(Qnet,self).__init__() - self.fc1=torch.nn.Linear(state_dim,hidden_dim) - self.fc2=torch.nn.Linear(hidden_dim,action_dim) - - - def forward(self,x): - x=F.relu(self.fc1(x)) - return self.fc2(x) - - -class DoubleDQN: - def __init__(self,state_dim,hidden_dim,action_dim): - if torch.cuda.is_available(): - self.device=torch.device('cuda') - else: - self.device=torch.device('cpu') - self.nn=Qnet(state_dim,hidden_dim,action_dim).to(self.device) - self.target_q_net=Qnet(state_dim,hidden_dim,action_dim).to(self.device) - self.optimizer=torch.optim.Adam(self.nn.parameters(),lr=2e-3) - self.genv=gym.make('CartPole-v0') - - - def env(self,a=None,initial=None): - if initial==True: - state=self.genv.reset(seed=0) - return state - else: - next_state,reward,done,_=self.genv.step(a) - return next_state,reward,done - - - def loss(self,s,a,next_s,r,d): - s=torch.tensor(s,dtype=torch.float).to(self.device) - a=torch.tensor(a,dtype=torch.int64).view(-1,1).to(self.device) - next_s=torch.tensor(next_s,dtype=torch.float).to(self.device) - r=torch.tensor(r,dtype=torch.float).view(-1,1).to(self.device) - d=torch.tensor(d,dtype=torch.float).view(-1,1).to(self.device) - q_value=self.nn(s).gather(1,a) - max_action=self.nn(next_s).max(1)[1].view(-1,1) - next_q_value=self.target_q_net(next_s).gather(1,max_action) - target=r+0.98*next_q_value*(1-d) - return F.mse_loss(q_value,target) - - - def backward(self,loss): - self.optimizer.zero_grad() - loss.backward() - return - - - def opt(self): - self.optimizer.step() - return - - - def update_param(self): - self.target_q_net.load_state_dict(self.nn.state_dict()) - return \ No newline at end of file diff --git a/7.0/models/RL/RL/pytorch/non_parallel/DuelingDQN.py b/7.0/models/RL/RL/pytorch/non_parallel/DuelingDQN.py deleted file mode 100644 index b81eeb37..00000000 --- a/7.0/models/RL/RL/pytorch/non_parallel/DuelingDQN.py +++ /dev/null @@ -1,67 +0,0 @@ -import torch -import gym -import torch.nn.functional as F - - -class VAnet(torch.nn.Module): - def __init__(self,state_dim,hidden_dim,action_dim): - super(VAnet,self).__init__() - self.fc1=torch.nn.Linear(state_dim,hidden_dim) - self.fc_A=torch.nn.Linear(hidden_dim,action_dim) - self.fc_V=torch.nn.Linear(hidden_dim,1) - - - def forward(self,x): - A=self.fc_A(F.relu(self.fc1(x))) - V=self.fc_V(F.relu(self.fc1(x))) - Q=V+A-A.mean(1).view(-1,1) - return Q - - -class DuelingDQN: - def __init__(self,state_dim,hidden_dim,action_dim): - if torch.cuda.is_available(): - self.device=torch.device('cuda') - else: - self.device=torch.device('cpu') - self.nn=VAnet(state_dim,hidden_dim,action_dim).to(self.device) - self.target_q_net=VAnet(state_dim,hidden_dim,action_dim).to(self.device) - self.optimizer=torch.optim.Adam(self.nn.parameters(),lr=2e-3) - self.genv=gym.make('CartPole-v0') - - - def env(self,a=None,initial=None): - if initial==True: - state=self.genv.reset(seed=0) - return state - else: - next_state,reward,done,_=self.genv.step(a) - return next_state,reward,done - - - def loss(self,s,a,next_s,r,d): - s=torch.tensor(s,dtype=torch.float).to(self.device) - a=torch.tensor(a,dtype=torch.int64).view(-1,1).to(self.device) - next_s=torch.tensor(next_s,dtype=torch.float).to(self.device) - r=torch.tensor(r,dtype=torch.float).view(-1,1).to(self.device) - d=torch.tensor(d,dtype=torch.float).view(-1,1).to(self.device) - q_value=self.nn(s).gather(1,a) - next_q_value=self.target_q_net(next_s).max(1)[0].view(-1,1) - target=r+0.98*next_q_value*(1-d) - return F.mse_loss(q_value,target) - - - def backward(self,loss): - self.optimizer.zero_grad() - loss.backward() - return - - - def opt(self): - self.optimizer.step() - return - - - def update_param(self): - self.target_q_net.load_state_dict(self.nn.state_dict()) - return \ No newline at end of file diff --git a/7.0/models/RL/RL/pytorch/parallel/DQN.py b/7.0/models/RL/RL/pytorch/parallel/DQN.py deleted file mode 100644 index f4d8417d..00000000 --- a/7.0/models/RL/RL/pytorch/parallel/DQN.py +++ /dev/null @@ -1,65 +0,0 @@ -import torch -import gym -import torch.nn.functional as F -from Note.nn.parallel.assign_device_pytorch import assign_device - - -class Qnet(torch.nn.Module): - def __init__(self,state_dim,hidden_dim,action_dim): - super(Qnet,self).__init__() - self.fc1=torch.nn.Linear(state_dim,hidden_dim) - self.fc2=torch.nn.Linear(hidden_dim,action_dim) - - - def forward(self,x): - x=F.relu(self.fc1(x)) - return self.fc2(x) - - -class DQN: - def __init__(self,state_dim,hidden_dim,action_dim): - if torch.cuda.is_available(): - self.device=torch.device('cuda') - else: - self.device=torch.device('cpu') - self.nn=Qnet(state_dim,hidden_dim,action_dim).to(self.device) - self.target_q_net=Qnet(state_dim,hidden_dim,action_dim).to(self.device) - self.optimizer=[torch.optim.Adam(self.nn.parameters(),lr=2e-3) for _ in range(5)] #optimizer,kernel uses it to optimize. - self.genv=[gym.make('CartPole-v0') for _ in range(5)] #create environment - - - def env(self,a=None,p=None,initial=None): #environment function,kernel uses it to interact with the environment. - if initial==True: - state=self.genv[p].reset() - return state - else: - next_state,reward,done,_=self.genv[p].step(a) - return next_state,reward,done - - - def loss(self,s,a,next_s,r,d,p): #loss function,kernel uses it to calculate loss. - s=torch.tensor(s,dtype=torch.float).to(assign_device(p,'GPU')) - a=torch.tensor(a,dtype=torch.int64).view(-1,1).to(assign_device(p,'GPU')) - next_s=torch.tensor(next_s,dtype=torch.float).to(assign_device(p,'GPU')) - r=torch.tensor(r,dtype=torch.float).view(-1,1).to(assign_device(p,'GPU')) - d=torch.tensor(d,dtype=torch.float).view(-1,1).to(assign_device(p,'GPU')) - q_value=self.nn(s).gather(1,a) - next_q_value=self.target_q_net(next_s).max(1)[0].view(-1,1) - target=r+0.98*next_q_value*(1-d) - return F.mse_loss(q_value,target) - - - def backward(self,loss,p): #backward function,kernel uses it for backpropagation. - self.optimizer[p].zero_grad(set_to_none=True) - loss.backward() - return - - - def opt(self,p): #opt function,kernel uses it to optimize. - self.optimizer[p].step() - return - - - def update_param(self): #update function,kernel uses it to update parameter. - self.target_q_net.load_state_dict(self.nn.state_dict()) - return \ No newline at end of file diff --git a/7.0/models/RL/RL/tensorflow/non_parallel/DDPG.py b/7.0/models/RL/RL/tensorflow/non_parallel/DDPG.py deleted file mode 100644 index 6474489a..00000000 --- a/7.0/models/RL/RL/tensorflow/non_parallel/DDPG.py +++ /dev/null @@ -1,76 +0,0 @@ -import tensorflow as tf # import TensorFlow library -from Note import nn -import gym # import OpenAI Gym library - - -class actor: # define a class for the actor network - def __init__(self,state_dim,hidden_dim,action_dim,action_bound): # initialize the network with state dimension, hidden dimension, action dimension and action bound - self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') - self.dense2 = nn.dense(action_dim, hidden_dim, activation='tanh') - self.action_bound=action_bound - self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list - - - def fp(self,x): # forward propagation function, kernel uses it for forward propagation - x = self.dense1(x) - return self.dense2(x)*self.action_bound - - -class critic: # define a class for the critic network - def __init__(self,state_dim,hidden_dim,action_dim): # initialize the network with state dimension, hidden dimension and action dimension - self.dense1 = nn.dense(hidden_dim, state_dim+action_dim, activation='relu') - self.dense2 = nn.dense(action_dim, hidden_dim) - self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list - - - def fp(self,x,a): # forward propagation function, kernel uses it for forward propagation - cat=tf.concat([x,a],axis=1) - x=self.dense1(cat) - return self.dense2(x) - - -class DDPG: # define a class for the DDPG agent - def __init__(self,hidden_dim,sigma,gamma,tau,actor_lr,critic_lr): # initialize the agent with hidden dimension, noise scale, discount factor, soft update factor, actor learning rate and critic learning rate - self.genv=gym.make('Pendulum-v1') # create environment - state_dim=self.genv.observation_space.shape[0] # get the state dimension from the environment observation space - action_dim=self.genv.action_space.shape[0] # get the action dimension from the environment action space - action_bound=self.genv.action_space.high[0] # get the action bound from the environment action space high value (assume symmetric) - self.actor=actor(state_dim,hidden_dim,action_dim,action_bound) # create an actor network for the agent - self.critic=critic(state_dim,hidden_dim,action_dim) # create a critic network for the agent - self.target_actor=actor(state_dim,hidden_dim,action_dim,action_bound) # create a target actor network for the agent - self.target_critic=critic(state_dim,hidden_dim,action_dim) # create a target critic network for the agent - self.actor_param=self.actor.param # parameter list of actor network, kernel uses it list for backpropagation - self.critic_param=self.critic.param # parameter list of critic network, kernel uses it list for backpropagation - nn.assign_param(self.target_actor.param,self.actor.param) # copy the parameters from actor network to target actor network - nn.assign_param(self.target_critic.param,self.critic.param) # copy the parameters from critic network to target critic network - self.param=[self.actor_param,self.critic_param] # parameter list of both networks, kernel uses it list for backpropagation - self.sigma=sigma # noise scale - self.gamma=gamma # discount factor - self.tau=tau # soft update factor - self.opt=tf.keras.optimizers.Adam() # optimizer, kernel uses it to optimize. Here we use Adam optimizer - - - def env(self,a=None,initial=None): # environment function, kernel uses it to interact with the environment - if initial==True: # if initial is True, reset the environment and return the initial state - state=self.genv.reset(seed=0) - return state - else: # otherwise, take an action and return the next state, reward and done flag - next_state,reward,done,_=self.genv.step(a) - return next_state,reward,done - - - def loss(self,s,a,next_s,r,d): # loss function, kernel uses it to calculate loss - a=tf.expand_dims(a,axis=1) # expand the action vector to match the Q-value matrix shape - next_q_value=self.target_critic.fp(next_s,self.target_actor.fp(next_s)) # get the Q-value for the next state and action from the target critic network - q_target=tf.cast(r,'float32')+self.gamma*next_q_value*(1-tf.cast(d,'float32')) # calculate the target value using Bellman equation with discount factor gamma - actor_loss=-tf.reduce_mean(self.critic.fp(s,self.actor.fp(s))) # calculate the actor loss as the negative mean of the Q-value for the current state and action from the critic network - critic_loss=tf.reduce_mean((self.critic.fp(s,a)-q_target)**2) # calculate the critic loss as the mean squared error between Q-value and target value - return [actor_loss,critic_loss] # return a list of actor loss and critic loss - - - def update_param(self): # update function, kernel uses it to update parameter - for target_param,param in zip(self.target_actor.param,self.actor.param): # for each pair of parameters in target actor network and actor network - target_param.assign(target_param*(1.0-self.tau)+param*self.tau) # update the target parameter using soft update with factor tau - for target_param,param in zip(self.target_critic.param,self.critic.param): # for each pair of parameters in target critic network and critic network - target_param.assign(target_param*(1.0-self.tau)+param*self.tau) # update the target parameter using soft update with factor tau - return \ No newline at end of file diff --git a/7.0/models/RL/RL/tensorflow/non_parallel/DQN.py b/7.0/models/RL/RL/tensorflow/non_parallel/DQN.py deleted file mode 100644 index d0be2840..00000000 --- a/7.0/models/RL/RL/tensorflow/non_parallel/DQN.py +++ /dev/null @@ -1,45 +0,0 @@ -import tensorflow as tf # import TensorFlow library -from Note import nn -import gym # import OpenAI Gym library - - -class Qnet: # define a class for the Q-network - def __init__(self,state_dim,hidden_dim,action_dim): # initialize the network with state dimension, hidden dimension and action dimension - self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') - self.dense2 = nn.dense(action_dim, hidden_dim) - self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list - - def fp(self,x): # forward propagation function, kernel uses it for forward propagation - x = self.dense2(self.dense1(x)) - return x - - -class DQN: # define a class for the DQN agent - def __init__(self,state_dim,hidden_dim,action_dim): # initialize the agent with state dimension, hidden dimension and action dimension - self.nn=Qnet(state_dim,hidden_dim,action_dim) # create a Q-network for the agent - self.target_q_net=Qnet(state_dim,hidden_dim,action_dim) # create a target Q-network for the agent - self.param=self.nn.param # parameter list, kernel uses it list for backpropagation - self.opt=tf.keras.optimizers.Adam() # optimizer, kernel uses it to optimize - self.genv=gym.make('CartPole-v0') # create environment - - - def env(self,a=None,initial=None): # environment function, kernel uses it to interact with the environment - if initial==True: # if initial is True, reset the environment and return the initial state - state=self.genv.reset(seed=0) - return state - else: # otherwise, take an action and return the next state, reward and done flag - next_state,reward,done,_=self.genv.step(a) - return next_state,reward,done - - - def loss(self,s,a,next_s,r,d): # loss function, kernel uses it to calculate loss - a=tf.expand_dims(a,axis=1) # expand the action vector to match the Q-value matrix shape - q_value=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) # get the Q-value for the selected action - next_q_value=tf.reduce_max(self.target_q_net.fp(next_s),axis=1) # get the maximum Q-value for the next state from the target network - target=tf.cast(r,'float32')+0.98*next_q_value*(1-tf.cast(d,'float32')) # calculate the target value using Bellman equation with discount factor 0.98 - return tf.reduce_mean((q_value-target)**2) # return the mean squared error between Q-value and target value - - - def update_param(self): # update function, kernel uses it to update parameter - nn.assign_param(self.target_q_net.param, self.param) # copy the parameters from the Q-network to the target network - return \ No newline at end of file diff --git a/7.0/models/RL/RL/tensorflow/non_parallel/DQN_pr.py b/7.0/models/RL/RL/tensorflow/non_parallel/DQN_pr.py deleted file mode 100644 index e907903a..00000000 --- a/7.0/models/RL/RL/tensorflow/non_parallel/DQN_pr.py +++ /dev/null @@ -1,58 +0,0 @@ -import tensorflow as tf -from Note import nn -import gym -import Note.RL.rl.prioritized_replay as pr - - -class Qnet: - def __init__(self,state_dim,hidden_dim,action_dim): - self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') - self.dense2 = nn.dense(action_dim, hidden_dim) - self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list - - def fp(self,x): - x = self.dense2(self.dense1(x)) - return x - - -class DQN: - def __init__(self,state_dim,hidden_dim,action_dim): - self.nn=Qnet(state_dim,hidden_dim,action_dim) - self.target_q_net=Qnet(state_dim,hidden_dim,action_dim) - self.param=self.nn.param - self.pr=pr.pr() - self.initial_TD=7 - self.epsilon=0.0007 - self.alpha=0.7 - self.opt=tf.keras.optimizers.Adam() - self.genv=gym.make('CartPole-v0') - - - def env(self,a=None,initial=None): - if initial==True: - state=self.genv.reset(seed=0) - return state - else: - next_state,reward,done,_=self.genv.step(a) - return next_state,reward,done - - - def data_func(self,state_pool,action_pool,next_state_pool,reward_pool,done_pool,batch): - s,a,next_s,r,d=self.pr.sample(state_pool,action_pool,next_state_pool,reward_pool,done_pool,self.epsilon,self.alpha,batch) - return s,a,next_s,r,d - - - def loss(self,s,a,next_s,r,d): - a=tf.expand_dims(a,axis=1) - q_value=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) - next_q_value=tf.reduce_max(self.target_q_net.fp(next_s),axis=1) - target=tf.cast(r,'float32')+0.98*next_q_value*(1-tf.cast(d,'float32')) - target=tf.expand_dims(target,axis=1) - TD=target-q_value - self.pr.update_TD(TD) - return tf.reduce_mean(TD**2) - - - def update_param(self): - nn.assign_param(self.target_q_net.param, self.param) - return \ No newline at end of file diff --git a/7.0/models/RL/RL/tensorflow/non_parallel/PPO.py b/7.0/models/RL/RL/tensorflow/non_parallel/PPO.py deleted file mode 100644 index dbce55f2..00000000 --- a/7.0/models/RL/RL/tensorflow/non_parallel/PPO.py +++ /dev/null @@ -1,68 +0,0 @@ -import tensorflow as tf -from Note import nn -import gym - - -class actor: - def __init__(self,state_dim,hidden_dim,action_dim): - self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') - self.dense2 = nn.dense(action_dim, hidden_dim) - self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list - - def fp(self,x): - x=self.dense1(x) - return tf.nn.softmax(self.dense2(x)) - - -class critic: - def __init__(self,state_dim,hidden_dim): - self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') - self.dense2 = nn.dense(1, hidden_dim) - self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list - - def fp(self,x): - x=self.dense1(x) - return self.dense2(x) - - -class PPO: - def __init__(self,state_dim,hidden_dim,action_dim,clip_eps,alpha): - self.actor=actor(state_dim,hidden_dim,action_dim) - self.nn=actor(state_dim,hidden_dim,action_dim) - nn.assign_param(self.nn.param,self.actor.param.copy()) - self.critic=critic(state_dim,hidden_dim) - self.clip_eps=clip_eps - self.alpha=alpha - self.param=[self.actor.param,self.critic.param] - self.opt=tf.keras.optimizers.Adam() - self.genv=gym.make('CartPole-v0') - - - def env(self,a=None,initial=None): - if initial==True: - state=self.genv.reset(seed=0) - return state - else: - next_state,reward,done,_=self.genv.step(a) - return next_state,reward,done - - - def loss(self,s,a,next_s,r,d): - a=tf.expand_dims(a,axis=1) - action_prob=tf.gather(self.actor.fp(s),a,axis=1,batch_dims=1) - action_prob_old=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) - raito=action_prob/action_prob_old - value=self.critic.fp(s) - value_tar=tf.cast(r,'float32')+0.98*self.critic.fp(next_s)*(1-tf.cast(d,'float32')) - TD=value_tar-value - sur1=raito*TD - sur2=tf.clip_by_value(raito,clip_value_min=1-self.clip_eps,clip_value_max=1+self.clip_eps)*TD - clip_loss=-tf.math.minimum(sur1,sur2) - entropy=action_prob*tf.math.log(action_prob+1e-8) - clip_loss=clip_loss-self.alpha*entropy - return [tf.reduce_mean(clip_loss),tf.reduce_mean((TD)**2)] - - - def update_param(self): - nn.assign_param(self.nn.param, self.actor.param) - return \ No newline at end of file diff --git a/7.0/models/RL/RL/tensorflow/non_parallel/PPO_eps.py b/7.0/models/RL/RL/tensorflow/non_parallel/PPO_eps.py deleted file mode 100644 index 899eccae..00000000 --- a/7.0/models/RL/RL/tensorflow/non_parallel/PPO_eps.py +++ /dev/null @@ -1,63 +0,0 @@ -import tensorflow as tf -from Note import nn -import gym - - -class actor: - def __init__(self,state_dim,hidden_dim,action_dim): - self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') - self.dense2 = nn.dense(action_dim, hidden_dim) - self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list - - def fp(self,x): - x=self.dense1(x) - return tf.nn.softmax(self.dense2(x)) - - -class critic: - def __init__(self,state_dim,hidden_dim): - self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') - self.dense2 = nn.dense(1, hidden_dim) - self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list - - def fp(self,x): - x=self.dense1(x) - return self.dense2(x) - - -class PPO: - def __init__(self,state_dim,hidden_dim,action_dim,clip_eps): - self.actor=actor(state_dim,hidden_dim,action_dim) - self.nn=actor(state_dim,hidden_dim,action_dim) - nn.assign_param(self.nn.param,self.actor.param.copy()) - self.critic=critic(state_dim,hidden_dim) - self.clip_eps=clip_eps - self.param=[self.actor.param,self.critic.param] - self.opt=tf.keras.optimizers.Adam() - self.genv=gym.make('CartPole-v0') - - - def env(self,a=None,initial=None): - if initial==True: - state=self.genv.reset(seed=0) - return state - else: - next_state,reward,done,_=self.genv.step(a) - return next_state,reward,done - - - def loss(self,s,a,next_s,r,d): - a=tf.expand_dims(a,axis=1) - raito=tf.gather(self.actor.fp(s),a,axis=1,batch_dims=1)/tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) - value=self.critic.fp(s) - value_tar=tf.cast(r,'float32')+0.98*self.critic.fp(next_s)*(1-tf.cast(d,'float32')) - TD=value_tar-value - sur1=raito*TD - sur2=tf.clip_by_value(raito,clip_value_min=1-self.clip_eps,clip_value_max=1+self.clip_eps)*TD - clip_loss=-tf.math.minimum(sur1,sur2) - return [tf.reduce_mean(clip_loss),tf.reduce_mean((TD)**2)] - - - def update_param(self): - nn.assign_param(self.nn.param, self.actor.param) - return \ No newline at end of file diff --git a/7.0/models/RL/RL/tensorflow/parallel/DDPG.py b/7.0/models/RL/RL/tensorflow/parallel/DDPG.py deleted file mode 100644 index 2b2f7d99..00000000 --- a/7.0/models/RL/RL/tensorflow/parallel/DDPG.py +++ /dev/null @@ -1,77 +0,0 @@ -import tensorflow as tf # import TensorFlow library -from Note import nn -import gym # import OpenAI Gym library -import Note.nn.parallel.optimizer as o # import Note's optimizer module - - -class actor: # define a class for the actor network - def __init__(self,state_dim,hidden_dim,action_dim,action_bound): # initialize the network with state dimension, hidden dimension, action dimension and action bound - self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') - self.dense2 = nn.dense(action_dim, hidden_dim, activation='tanh') - self.action_bound=action_bound - self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list - - - def fp(self,x): # forward propagation function, kernel uses it for forward propagation - x = self.dense1(x) - return self.dense2(x)*self.action_bound - - -class critic: # define a class for the critic network - def __init__(self,state_dim,hidden_dim,action_dim): # initialize the network with state dimension, hidden dimension and action dimension - self.dense1 = nn.dense(hidden_dim, state_dim+action_dim, activation='relu') - self.dense2 = nn.dense(action_dim, hidden_dim) - self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list - - - def fp(self,x,a): # forward propagation function, kernel uses it for forward propagation - cat=tf.concat([x,a],axis=1) - x=self.dense1(cat) - return self.dense2(x) - - -class DDPG: # define a class for the DDPG agent - def __init__(self,hidden_dim,sigma,gamma,tau,actor_lr,critic_lr): # initialize the agent with hidden dimension, noise scale, discount factor, soft update factor, actor learning rate and critic learning rate - self.genv=[gym.make('Pendulum-v1') for _ in range(5)] # create a list of 5 environments - state_dim=self.genv.observation_space.shape[0] # get the state dimension from the environment observation space - action_dim=self.genv.action_space.shape[0] # get the action dimension from the environment action space - action_bound=self.genv.action_space.high[0] # get the action bound from the environment action space high value (assume symmetric) - self.actor=actor(state_dim,hidden_dim,action_dim,action_bound) # create an actor network for the agent - self.critic=critic(state_dim,hidden_dim,action_dim) # create a critic network for the agent - self.target_actor=actor(state_dim,hidden_dim,action_dim,action_bound) # create a target actor network for the agent - self.target_critic=critic(state_dim,hidden_dim,action_dim) # create a target critic network for the agent - self.actor_param=self.actor.param # parameter list of actor network, kernel uses it list for backpropagation - self.critic_param=self.critic.param # parameter list of critic network, kernel uses it list for backpropagation - nn.assign_param(self.target_actor.param,self.actor.param) # copy the parameters from actor network to target actor network - nn.assign_param(self.target_critic.param,self.critic.param) # copy the parameters from critic network to target critic network - self.param=[self.actor_param,self.critic_param] # parameter list of both networks, kernel uses it list for backpropagation - self.sigma=sigma # noise scale - self.gamma=gamma # discount factor - self.tau=tau # soft update factor - self.optimizer=o.SGD(param=self.param) # optimizer, kernel uses it to optimize. Here we use a custom SGD optimizer - - - def env(self,a=None,p=None,initial=None): # environment function, kernel uses it to interact with the environment - if initial==True: # if initial is True, reset the environment with index p and return the initial state - state=self.genv[p].reset(seed=0) - return state - else: # otherwise, take an action and return the next state, reward and done flag from the environment with index p - next_state,reward,done,_=self.genv[p].step(a) - return next_state,reward,done - - - def loss(self,s,a,next_s,r,d): # loss function, kernel uses it to calculate loss - a=tf.expand_dims(a,axis=1) # expand the action vector to match the Q-value matrix shape - next_q_value=self.target_critic.fp(next_s,self.target_actor.fp(next_s)) # get the Q-value for the next state and action from the target critic network - q_target=tf.cast(r,'float32')+self.gamma*next_q_value*(1-tf.cast(d,'float32')) # calculate the target value using Bellman equation with discount factor gamma - actor_loss=-tf.reduce_mean(self.critic.fp(s,self.actor.fp(s))) # calculate the actor loss as the negative mean of the Q-value for the current state and action from the critic network - critic_loss=tf.reduce_mean((self.critic.fp(s,a)-q_target)**2) # calculate the critic loss as the mean squared error between Q-value and target value - return [actor_loss,critic_loss] # return a list of actor loss and critic loss - - - def update_param(self): # update function, kernel uses it to update parameter - for target_param,param in zip(self.target_actor.param,self.actor.param): # for each pair of parameters in target actor network and actor network - target_param.assign(target_param*(1.0-self.tau)+param*self.tau) # update the target parameter using soft update with factor tau - for target_param,param in zip(self.target_critic.param,self.critic.param): # for each pair of parameters in target critic network and critic network - target_param.assign(target_param*(1.0-self.tau)+param*self.tau) # update the target parameter using soft update with factor tau - return \ No newline at end of file diff --git a/7.0/models/RL/RL/tensorflow/parallel/DQN.py b/7.0/models/RL/RL/tensorflow/parallel/DQN.py deleted file mode 100644 index 9186307d..00000000 --- a/7.0/models/RL/RL/tensorflow/parallel/DQN.py +++ /dev/null @@ -1,51 +0,0 @@ -import tensorflow as tf # import TensorFlow library -from Note import nn -import gym # import OpenAI Gym library -import Note.nn.parallel.optimizer as o # import Note's optimizer module - - -class Qnet: # define a class for the Q-network - def __init__(self,state_dim,hidden_dim,action_dim): # initialize the network with state dimension, hidden dimension and action dimension - self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') - self.dense2 = nn.dense(action_dim, hidden_dim) - self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list - - def fp(self,x): # forward propagation function, kernel uses it for forward propagation - x = self.dense2(self.dense1(x)) - return x - - -class DQN: # define a class for the DQN agent - def __init__(self,state_dim,hidden_dim,action_dim): # initialize the agent with state dimension, hidden dimension and action dimension - self.nn=Qnet(state_dim,hidden_dim,action_dim) # create a Q-network for the agent - self.target_q_net=Qnet(state_dim,hidden_dim,action_dim) # create a target Q-network for the agent - self.param=self.nn.param # parameter list, kernel uses it list for backpropagation - self.optimizer=o.SGD(param=self.param) # optimizer, kernel uses it to optimize. Here we use a custom SGD optimizer - self.genv=[gym.make('CartPole-v0') for _ in range(5)] # create a list of 5 environments - - - def env(self,a=None,p=None,initial=None): # environment function, kernel uses it to interact with the environment - if initial==True: # if initial is True, reset the environment with index p and return the initial state - state=self.genv[p].reset(seed=0) - return state - else: # otherwise, take an action and return the next state, reward and done flag from the environment with index p - next_state,reward,done,_=self.genv[p].step(a) - return next_state,reward,done - - - def loss(self,s,a,next_s,r,d): # loss function, kernel uses it to calculate loss - a=tf.expand_dims(a,axis=1) # expand the action vector to match the Q-value matrix shape - q_value=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) # get the Q-value for the selected action - next_q_value=tf.reduce_max(self.target_q_net.fp(next_s),axis=1) # get the maximum Q-value for the next state from the target network - target=tf.cast(r,'float32')+0.98*next_q_value*(1-tf.cast(d,'float32')) # calculate the target value using Bellman equation with discount factor 0.98 - return tf.reduce_mean((q_value-target)**2) # return the mean squared error between Q-value and target value - - - def update_param(self): # update function, kernel uses it to update parameter - nn.assign(self.target_q_net.param,self.param) # copy the parameters from the Q-network to the target network - return - - - def opt(self,gradient): # optimization function, kernel uses it to optimize parameter - param=self.optimizer.opt(gradient,self.param) # apply the custom momentum optimizer to update the parameters using the gradient - return param \ No newline at end of file diff --git a/7.0/models/RL/RL/tensorflow/parallel/PPO.py b/7.0/models/RL/RL/tensorflow/parallel/PPO.py deleted file mode 100644 index dbf28400..00000000 --- a/7.0/models/RL/RL/tensorflow/parallel/PPO.py +++ /dev/null @@ -1,68 +0,0 @@ -import tensorflow as tf -from Note import nn -import gym -from Note.nn.parallel.optimizer import SGD - - -class actor: - def __init__(self,state_dim,hidden_dim,action_dim): - self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') - self.dense2 = nn.dense(action_dim, hidden_dim) - self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list - - def fp(self,x): - x=self.dense1(x) - return tf.nn.softmax(self.dense2(x)) - - -class critic: - def __init__(self,state_dim,hidden_dim): - self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') - self.dense2 = nn.dense(1, hidden_dim) - self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list - - def fp(self,x): - x=self.dense1(x) - return self.dense2(x) - - -class PPO: - def __init__(self,state_dim,hidden_dim,action_dim,clip_eps,alpha): - self.actor=actor(state_dim,hidden_dim,action_dim) - self.nn=actor(state_dim,hidden_dim,action_dim) - nn.assign(self.nn.param,self.actor.param.copy()) - self.critic=critic(state_dim,hidden_dim) - self.clip_eps=clip_eps - self.param=[self.actor.param,self.critic.param] - self.opt=SGD(param=self.param) - self.genv=[gym.make('CartPole-v0') for _ in range(5)] - - - def env(self,a=None,p=None,initial=None): - if initial==True: - state=self.genv[p].reset(seed=0) - return state - else: - next_state,reward,done,_=self.genv[p].step(a) - return next_state,reward,done - - - def loss(self,s,a,next_s,r,d): - a=tf.expand_dims(a,axis=1) - action_prob=tf.gather(self.actor.fp(s),a,axis=1,batch_dims=1) - action_prob_old=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) - raito=action_prob/action_prob_old - value=self.critic.fp(s) - value_tar=tf.cast(r,'float32')+0.98*self.critic.fp(next_s)*(1-tf.cast(d,'float32')) - TD=value_tar-value - sur1=raito*TD - sur2=tf.clip_by_value(raito,clip_value_min=1-self.clip_eps,clip_value_max=1+self.clip_eps)*TD - clip_loss=-tf.math.minimum(sur1,sur2) - entropy=action_prob*tf.math.log(action_prob+1e-8) - clip_loss=clip_loss-self.alpha*entropy - return [tf.reduce_mean(clip_loss),tf.reduce_mean((TD)**2)] - - - def update_param(self): - nn.assign(self.nn.param,self.actor.param) - return \ No newline at end of file diff --git a/7.0/models/RL/RL/tensorflow/parallel/PPO_eps.py b/7.0/models/RL/RL/tensorflow/parallel/PPO_eps.py deleted file mode 100644 index 4a4eb6af..00000000 --- a/7.0/models/RL/RL/tensorflow/parallel/PPO_eps.py +++ /dev/null @@ -1,64 +0,0 @@ -import tensorflow as tf -from Note import nn -import gym -from Note.nn.parallel.optimizer import SGD - - -class actor: - def __init__(self,state_dim,hidden_dim,action_dim): - self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') - self.dense2 = nn.dense(action_dim, hidden_dim) - self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list - - def fp(self,x): - x=self.dense1(x) - return tf.nn.softmax(self.dense2(x)) - - -class critic: - def __init__(self,state_dim,hidden_dim): - self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') - self.dense2 = nn.dense(1, hidden_dim) - self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list - - def fp(self,x): - x=self.dense1(x) - return self.dense2(x) - - -class PPO: - def __init__(self,state_dim,hidden_dim,action_dim,clip_eps): - self.actor=actor(state_dim,hidden_dim,action_dim) - self.nn=actor(state_dim,hidden_dim,action_dim) - nn.assign(self.nn.param,self.actor.param.copy()) - self.critic=critic(state_dim,hidden_dim) - self.clip_eps=clip_eps - self.param=[self.actor.param,self.critic.param] - self.opt=SGD(param=self.param) - self.genv=[gym.make('CartPole-v0') for _ in range(5)] - - - def env(self,a=None,p=None,initial=None): - if initial==True: - state=self.genv[p].reset(seed=0) - return state - else: - next_state,reward,done,_=self.genv[p].step(a) - return next_state,reward,done - - - def loss(self,s,a,next_s,r,d): - a=tf.expand_dims(a,axis=1) - raito=tf.gather(self.actor.fp(s),a,axis=1,batch_dims=1)/tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) - value=self.critic.fp(s) - value_tar=tf.cast(r,'float32')+0.98*self.critic.fp(next_s)*(1-tf.cast(d,'float32')) - TD=value_tar-value - sur1=raito*TD - sur2=tf.clip_by_value(raito,clip_value_min=1-self.clip_eps,clip_value_max=1+self.clip_eps)*TD - clip_loss=-tf.math.minimum(sur1,sur2) - return [tf.reduce_mean(clip_loss),tf.reduce_mean((TD)**2)] - - - def update_param(self): - nn.assign(self.nn.param,self.actor.param) - return \ No newline at end of file From 28026009775c734e55a6341c890442593b4e3d7f Mon Sep 17 00:00:00 2001 From: NoteDance Date: Thu, 19 Sep 2024 15:30:02 +0800 Subject: [PATCH 321/337] Add files via upload --- 7.0/models/RL/tensorflow/non_parallel/DDPG.py | 4 ++-- 7.0/models/RL/tensorflow/non_parallel/PPO.py | 2 +- 7.0/models/RL/tensorflow/non_parallel/PPO_eps.py | 2 +- 7.0/models/RL/tensorflow/parallel/DDPG.py | 4 ++-- 7.0/models/RL/tensorflow/parallel/PPO.py | 2 +- 7.0/models/RL/tensorflow/parallel/PPO_eps.py | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/7.0/models/RL/tensorflow/non_parallel/DDPG.py b/7.0/models/RL/tensorflow/non_parallel/DDPG.py index ee6ed007..6474489a 100644 --- a/7.0/models/RL/tensorflow/non_parallel/DDPG.py +++ b/7.0/models/RL/tensorflow/non_parallel/DDPG.py @@ -65,7 +65,7 @@ def loss(self,s,a,next_s,r,d): # loss function, kernel uses it to calculate los q_target=tf.cast(r,'float32')+self.gamma*next_q_value*(1-tf.cast(d,'float32')) # calculate the target value using Bellman equation with discount factor gamma actor_loss=-tf.reduce_mean(self.critic.fp(s,self.actor.fp(s))) # calculate the actor loss as the negative mean of the Q-value for the current state and action from the critic network critic_loss=tf.reduce_mean((self.critic.fp(s,a)-q_target)**2) # calculate the critic loss as the mean squared error between Q-value and target value - return actor_loss+critic_loss # return a list of actor loss and critic loss + return [actor_loss,critic_loss] # return a list of actor loss and critic loss def update_param(self): # update function, kernel uses it to update parameter @@ -73,4 +73,4 @@ def update_param(self): # update function, kernel uses it to update parameter target_param.assign(target_param*(1.0-self.tau)+param*self.tau) # update the target parameter using soft update with factor tau for target_param,param in zip(self.target_critic.param,self.critic.param): # for each pair of parameters in target critic network and critic network target_param.assign(target_param*(1.0-self.tau)+param*self.tau) # update the target parameter using soft update with factor tau - return + return \ No newline at end of file diff --git a/7.0/models/RL/tensorflow/non_parallel/PPO.py b/7.0/models/RL/tensorflow/non_parallel/PPO.py index c0e9b704..dbce55f2 100644 --- a/7.0/models/RL/tensorflow/non_parallel/PPO.py +++ b/7.0/models/RL/tensorflow/non_parallel/PPO.py @@ -60,7 +60,7 @@ def loss(self,s,a,next_s,r,d): clip_loss=-tf.math.minimum(sur1,sur2) entropy=action_prob*tf.math.log(action_prob+1e-8) clip_loss=clip_loss-self.alpha*entropy - return tf.reduce_mean(clip_loss)+tf.reduce_mean((TD)**2) + return [tf.reduce_mean(clip_loss),tf.reduce_mean((TD)**2)] def update_param(self): diff --git a/7.0/models/RL/tensorflow/non_parallel/PPO_eps.py b/7.0/models/RL/tensorflow/non_parallel/PPO_eps.py index 1287521b..899eccae 100644 --- a/7.0/models/RL/tensorflow/non_parallel/PPO_eps.py +++ b/7.0/models/RL/tensorflow/non_parallel/PPO_eps.py @@ -55,7 +55,7 @@ def loss(self,s,a,next_s,r,d): sur1=raito*TD sur2=tf.clip_by_value(raito,clip_value_min=1-self.clip_eps,clip_value_max=1+self.clip_eps)*TD clip_loss=-tf.math.minimum(sur1,sur2) - return tf.reduce_mean(clip_loss)+tf.reduce_mean((TD)**2) + return [tf.reduce_mean(clip_loss),tf.reduce_mean((TD)**2)] def update_param(self): diff --git a/7.0/models/RL/tensorflow/parallel/DDPG.py b/7.0/models/RL/tensorflow/parallel/DDPG.py index 8209ad26..2b2f7d99 100644 --- a/7.0/models/RL/tensorflow/parallel/DDPG.py +++ b/7.0/models/RL/tensorflow/parallel/DDPG.py @@ -66,7 +66,7 @@ def loss(self,s,a,next_s,r,d): # loss function, kernel uses it to calculate los q_target=tf.cast(r,'float32')+self.gamma*next_q_value*(1-tf.cast(d,'float32')) # calculate the target value using Bellman equation with discount factor gamma actor_loss=-tf.reduce_mean(self.critic.fp(s,self.actor.fp(s))) # calculate the actor loss as the negative mean of the Q-value for the current state and action from the critic network critic_loss=tf.reduce_mean((self.critic.fp(s,a)-q_target)**2) # calculate the critic loss as the mean squared error between Q-value and target value - return actor_loss+critic_loss # return a list of actor loss and critic loss + return [actor_loss,critic_loss] # return a list of actor loss and critic loss def update_param(self): # update function, kernel uses it to update parameter @@ -74,4 +74,4 @@ def update_param(self): # update function, kernel uses it to update parameter target_param.assign(target_param*(1.0-self.tau)+param*self.tau) # update the target parameter using soft update with factor tau for target_param,param in zip(self.target_critic.param,self.critic.param): # for each pair of parameters in target critic network and critic network target_param.assign(target_param*(1.0-self.tau)+param*self.tau) # update the target parameter using soft update with factor tau - return + return \ No newline at end of file diff --git a/7.0/models/RL/tensorflow/parallel/PPO.py b/7.0/models/RL/tensorflow/parallel/PPO.py index 88763150..dbf28400 100644 --- a/7.0/models/RL/tensorflow/parallel/PPO.py +++ b/7.0/models/RL/tensorflow/parallel/PPO.py @@ -60,7 +60,7 @@ def loss(self,s,a,next_s,r,d): clip_loss=-tf.math.minimum(sur1,sur2) entropy=action_prob*tf.math.log(action_prob+1e-8) clip_loss=clip_loss-self.alpha*entropy - return tf.reduce_mean(clip_loss)+tf.reduce_mean((TD)**2) + return [tf.reduce_mean(clip_loss),tf.reduce_mean((TD)**2)] def update_param(self): diff --git a/7.0/models/RL/tensorflow/parallel/PPO_eps.py b/7.0/models/RL/tensorflow/parallel/PPO_eps.py index 008baf04..4a4eb6af 100644 --- a/7.0/models/RL/tensorflow/parallel/PPO_eps.py +++ b/7.0/models/RL/tensorflow/parallel/PPO_eps.py @@ -56,7 +56,7 @@ def loss(self,s,a,next_s,r,d): sur1=raito*TD sur2=tf.clip_by_value(raito,clip_value_min=1-self.clip_eps,clip_value_max=1+self.clip_eps)*TD clip_loss=-tf.math.minimum(sur1,sur2) - return tf.reduce_mean(clip_loss)+tf.reduce_mean((TD)**2) + return [tf.reduce_mean(clip_loss),tf.reduce_mean((TD)**2)] def update_param(self): From d0013bdc9ea602b4d8d4b4f1f044ed56f71324c1 Mon Sep 17 00:00:00 2001 From: NoteDance Date: Tue, 24 Sep 2024 22:24:42 +0800 Subject: [PATCH 322/337] Update DQN_pr.py --- 7.0/models/RL/tensorflow/non_parallel/DQN_pr.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/7.0/models/RL/tensorflow/non_parallel/DQN_pr.py b/7.0/models/RL/tensorflow/non_parallel/DQN_pr.py index e907903a..c9832b1d 100644 --- a/7.0/models/RL/tensorflow/non_parallel/DQN_pr.py +++ b/7.0/models/RL/tensorflow/non_parallel/DQN_pr.py @@ -21,7 +21,7 @@ def __init__(self,state_dim,hidden_dim,action_dim): self.target_q_net=Qnet(state_dim,hidden_dim,action_dim) self.param=self.nn.param self.pr=pr.pr() - self.initial_TD=7 + self.initial_TD=tf.Variable(7.) self.epsilon=0.0007 self.alpha=0.7 self.opt=tf.keras.optimizers.Adam() @@ -55,4 +55,4 @@ def loss(self,s,a,next_s,r,d): def update_param(self): nn.assign_param(self.target_q_net.param, self.param) - return \ No newline at end of file + return From a640a9068a89d923d231299555abfe5bd3900bce Mon Sep 17 00:00:00 2001 From: NoteDance Date: Tue, 24 Sep 2024 22:25:03 +0800 Subject: [PATCH 323/337] Update DQN_pr.py --- 7.0/models/RL/pytorch/non_parallel/DQN_pr.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/7.0/models/RL/pytorch/non_parallel/DQN_pr.py b/7.0/models/RL/pytorch/non_parallel/DQN_pr.py index 66c71847..763a47fd 100644 --- a/7.0/models/RL/pytorch/non_parallel/DQN_pr.py +++ b/7.0/models/RL/pytorch/non_parallel/DQN_pr.py @@ -25,7 +25,7 @@ def __init__(self,state_dim,hidden_dim,action_dim): self.nn=Qnet(state_dim,hidden_dim,action_dim).to(self.device) self.target_q_net=Qnet(state_dim,hidden_dim,action_dim).to(self.device) self.pr=pr.pr() - self.initial_TD=7 + self.initial_TD=7. self._epsilon=0.0007 self.alpha=0.7 self.optimizer=torch.optim.Adam(self.nn.parameters(),lr=2e-3) @@ -73,4 +73,4 @@ def opt(self): def update_param(self): self.target_q_net.load_state_dict(self.nn.state_dict()) - return \ No newline at end of file + return From 0fabfb04e087cdbd6a0a499f153df77d396bc937 Mon Sep 17 00:00:00 2001 From: NoteDance Date: Mon, 21 Oct 2024 21:33:19 +0800 Subject: [PATCH 324/337] Add files via upload --- 7.0/models/RL/tensorflow/parallel/DQN_pr.py | 61 +++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 7.0/models/RL/tensorflow/parallel/DQN_pr.py diff --git a/7.0/models/RL/tensorflow/parallel/DQN_pr.py b/7.0/models/RL/tensorflow/parallel/DQN_pr.py new file mode 100644 index 00000000..e8d58a78 --- /dev/null +++ b/7.0/models/RL/tensorflow/parallel/DQN_pr.py @@ -0,0 +1,61 @@ +import tensorflow as tf # import TensorFlow library +from Note import nn +import Note.RL.rl.prioritized_replay as pr +import gym # import OpenAI Gym library +import Note.nn.parallel.optimizer as o # import Note's optimizer module + + +class Qnet: # define a class for the Q-network + def __init__(self,state_dim,hidden_dim,action_dim): # initialize the network with state dimension, hidden dimension and action dimension + self.dense1 = nn.dense(hidden_dim, state_dim, activation='relu') + self.dense2 = nn.dense(action_dim, hidden_dim) + self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list + + def fp(self,x): # forward propagation function, kernel uses it for forward propagation + x = self.dense2(self.dense1(x)) + return x + + +class DQN: # define a class for the DQN agent + def __init__(self,state_dim,hidden_dim,action_dim): # initialize the agent with state dimension, hidden dimension and action dimension + self.nn=Qnet(state_dim,hidden_dim,action_dim) # create a Q-network for the agent + self.target_q_net=Qnet(state_dim,hidden_dim,action_dim) # create a target Q-network for the agent + self.param=self.nn.param # parameter list, kernel uses it list for backpropagation + self.pr=pr.pr_mp() + self.initial_TD=tf.Variable(7.) + self.epsilon=0.0007 + self.alpha=0.7 + self.optimizer=o.SGD(param=self.param) # optimizer, kernel uses it to optimize. Here we use a custom SGD optimizer + self.genv=[gym.make('CartPole-v0') for _ in range(5)] # create a list of 5 environments + + + def env(self,a=None,p=None,initial=None): # environment function, kernel uses it to interact with the environment + if initial==True: # if initial is True, reset the environment with index p and return the initial state + state=self.genv[p].reset(seed=0) + return state + else: # otherwise, take an action and return the next state, reward and done flag from the environment with index p + next_state,reward,done,_=self.genv[p].step(a) + return next_state,reward,done + + + def data_func(self,state_pool,action_pool,next_state_pool,reward_pool,done_pool,batch,p): + s,a,next_s,r,d=self.pr.sample(state_pool,action_pool,next_state_pool,reward_pool,done_pool,self.epsilon,self.alpha,batch,p) + return s,a,next_s,r,d + + + def loss(self,s,a,next_s,r,d): # loss function, kernel uses it to calculate loss + a=tf.expand_dims(a,axis=1) # expand the action vector to match the Q-value matrix shape + q_value=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) # get the Q-value for the selected action + next_q_value=tf.reduce_max(self.target_q_net.fp(next_s),axis=1) # get the maximum Q-value for the next state from the target network + target=tf.cast(r,'float32')+0.98*next_q_value*(1-tf.cast(d,'float32')) # calculate the target value using Bellman equation with discount factor 0.98 + return tf.reduce_mean((q_value-target)**2) # return the mean squared error between Q-value and target value + + + def update_param(self): # update function, kernel uses it to update parameter + nn.assign(self.target_q_net.param,self.param) # copy the parameters from the Q-network to the target network + return + + + def opt(self,gradient): # optimization function, kernel uses it to optimize parameter + param=self.optimizer.opt(gradient,self.param) # apply the custom momentum optimizer to update the parameters using the gradient + return param \ No newline at end of file From 51aa18b5cb67af4d289cd9ab94bc26d5237536e1 Mon Sep 17 00:00:00 2001 From: NoteDance Date: Wed, 23 Oct 2024 21:55:24 +0800 Subject: [PATCH 325/337] Add files via upload --- 7.0/models/RL/pytorch/parallel/DQN_pr.py | 75 ++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 7.0/models/RL/pytorch/parallel/DQN_pr.py diff --git a/7.0/models/RL/pytorch/parallel/DQN_pr.py b/7.0/models/RL/pytorch/parallel/DQN_pr.py new file mode 100644 index 00000000..e2cb1818 --- /dev/null +++ b/7.0/models/RL/pytorch/parallel/DQN_pr.py @@ -0,0 +1,75 @@ +import torch +import Note.RL.rl.prioritized_replay as pr +import gym +import torch.nn.functional as F +from Note.nn.parallel.assign_device_pytorch import assign_device + + +class Qnet(torch.nn.Module): + def __init__(self,state_dim,hidden_dim,action_dim): + super(Qnet,self).__init__() + self.fc1=torch.nn.Linear(state_dim,hidden_dim) + self.fc2=torch.nn.Linear(hidden_dim,action_dim) + + + def forward(self,x): + x=F.relu(self.fc1(x)) + return self.fc2(x) + + +class DQN: + def __init__(self,state_dim,hidden_dim,action_dim): + if torch.cuda.is_available(): + self.device=torch.device('cuda') + else: + self.device=torch.device('cpu') + self.nn=Qnet(state_dim,hidden_dim,action_dim).to(self.device) + self.target_q_net=Qnet(state_dim,hidden_dim,action_dim).to(self.device) + self.pr=pr.pr_mp_() + self.initial_TD=7. + self.epsilon=0.0007 + self.alpha=0.7 + self.optimizer=[torch.optim.Adam(self.nn.parameters(),lr=2e-3) for _ in range(5)] #optimizer,kernel uses it to optimize. + self.genv=[gym.make('CartPole-v0') for _ in range(5)] #create environment + + + def env(self,a=None,p=None,initial=None): #environment function,kernel uses it to interact with the environment. + if initial==True: + state=self.genv[p].reset() + return state + else: + next_state,reward,done,_=self.genv[p].step(a) + return next_state,reward,done + + + def data_func(self,state_pool,action_pool,next_state_pool,reward_pool,done_pool,batch,p): + s,a,next_s,r,d=self.pr.sample(state_pool,action_pool,next_state_pool,reward_pool,done_pool,self.epsilon,self.alpha,batch,p) + return s,a,next_s,r,d + + + def loss(self,s,a,next_s,r,d,p): #loss function,kernel uses it to calculate loss. + s=torch.tensor(s,dtype=torch.float).to(assign_device(p,'GPU')) + a=torch.tensor(a,dtype=torch.int64).view(-1,1).to(assign_device(p,'GPU')) + next_s=torch.tensor(next_s,dtype=torch.float).to(assign_device(p,'GPU')) + r=torch.tensor(r,dtype=torch.float).view(-1,1).to(assign_device(p,'GPU')) + d=torch.tensor(d,dtype=torch.float).view(-1,1).to(assign_device(p,'GPU')) + q_value=self.nn(s).gather(1,a) + next_q_value=self.target_q_net(next_s).max(1)[0].view(-1,1) + target=r+0.98*next_q_value*(1-d) + return F.mse_loss(q_value,target) + + + def backward(self,loss,p): #backward function,kernel uses it for backpropagation. + self.optimizer[p].zero_grad(set_to_none=True) + loss.backward() + return + + + def opt(self,p): #opt function,kernel uses it to optimize. + self.optimizer[p].step() + return + + + def update_param(self): #update function,kernel uses it to update parameter. + self.target_q_net.load_state_dict(self.nn.state_dict()) + return \ No newline at end of file From f0ca54a1742aab764c2a9c8adb359269f194370c Mon Sep 17 00:00:00 2001 From: NoteDance Date: Thu, 24 Oct 2024 21:09:08 +0800 Subject: [PATCH 326/337] Update DQN_pr.py --- 7.0/models/RL/tensorflow/parallel/DQN_pr.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/7.0/models/RL/tensorflow/parallel/DQN_pr.py b/7.0/models/RL/tensorflow/parallel/DQN_pr.py index e8d58a78..16192390 100644 --- a/7.0/models/RL/tensorflow/parallel/DQN_pr.py +++ b/7.0/models/RL/tensorflow/parallel/DQN_pr.py @@ -48,7 +48,9 @@ def loss(self,s,a,next_s,r,d): # loss function, kernel uses it to calculate loss q_value=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) # get the Q-value for the selected action next_q_value=tf.reduce_max(self.target_q_net.fp(next_s),axis=1) # get the maximum Q-value for the next state from the target network target=tf.cast(r,'float32')+0.98*next_q_value*(1-tf.cast(d,'float32')) # calculate the target value using Bellman equation with discount factor 0.98 - return tf.reduce_mean((q_value-target)**2) # return the mean squared error between Q-value and target value + TD=q_value-target + self.pr.update_TD(TD) + return tf.reduce_mean(TD**2) # return the mean squared error between Q-value and target value def update_param(self): # update function, kernel uses it to update parameter @@ -58,4 +60,4 @@ def update_param(self): # update function, kernel uses it to update parameter def opt(self,gradient): # optimization function, kernel uses it to optimize parameter param=self.optimizer.opt(gradient,self.param) # apply the custom momentum optimizer to update the parameters using the gradient - return param \ No newline at end of file + return param From 88f270c64774d88dc5deb32d07a27f3464837eb8 Mon Sep 17 00:00:00 2001 From: NoteDance Date: Thu, 24 Oct 2024 21:09:28 +0800 Subject: [PATCH 327/337] Update DQN_pr.py --- 7.0/models/RL/pytorch/parallel/DQN_pr.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/7.0/models/RL/pytorch/parallel/DQN_pr.py b/7.0/models/RL/pytorch/parallel/DQN_pr.py index e2cb1818..53718dd9 100644 --- a/7.0/models/RL/pytorch/parallel/DQN_pr.py +++ b/7.0/models/RL/pytorch/parallel/DQN_pr.py @@ -56,7 +56,9 @@ def loss(self,s,a,next_s,r,d,p): #loss function,kernel uses it to calculate loss q_value=self.nn(s).gather(1,a) next_q_value=self.target_q_net(next_s).max(1)[0].view(-1,1) target=r+0.98*next_q_value*(1-d) - return F.mse_loss(q_value,target) + TD=target-q_value + self.pr.update_TD(TD) + return torch.mean(TD**2) def backward(self,loss,p): #backward function,kernel uses it for backpropagation. @@ -72,4 +74,4 @@ def opt(self,p): #opt function,kernel uses it to optimize. def update_param(self): #update function,kernel uses it to update parameter. self.target_q_net.load_state_dict(self.nn.state_dict()) - return \ No newline at end of file + return From b01712fd851735170d13ff50a39628b51cb430d7 Mon Sep 17 00:00:00 2001 From: NoteDance Date: Thu, 13 Feb 2025 18:32:06 +0800 Subject: [PATCH 328/337] Update DDPG.py --- 7.0/models/RL/pytorch/non_parallel/DDPG.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/7.0/models/RL/pytorch/non_parallel/DDPG.py b/7.0/models/RL/pytorch/non_parallel/DDPG.py index 93af197f..238a27d2 100644 --- a/7.0/models/RL/pytorch/non_parallel/DDPG.py +++ b/7.0/models/RL/pytorch/non_parallel/DDPG.py @@ -71,14 +71,13 @@ def loss(self,s,a,next_s,r,d): q_target=r+self.gamma*next_q_value*(1-d) actor_loss=-torch.mean(self.critic(s,self.actor(s))) critic_loss=F.mse_loss(self.critic(s,a),q_target) - return [actor_loss,critic_loss] + return actor_loss+critic_loss def backward(self,loss): self.actor_opt.zero_grad() - loss[0].backward() self.critic_opt.zero_grad() - loss[1].backward() + loss.backward() return @@ -93,4 +92,4 @@ def update_param(self): target_param.data.copy_(target_param.data*(1.0-self.tau)+param.data*self.tau) for target_param,param in zip(self.target_critic.parameters(),self.critic.parameters()): target_param.data.copy_(target_param.data*(1.0-self.tau)+param.data*self.tau) - return \ No newline at end of file + return From 7f81c6c6a4eff03e3f873d2490268a0966f683c1 Mon Sep 17 00:00:00 2001 From: NoteDance Date: Thu, 13 Feb 2025 18:35:01 +0800 Subject: [PATCH 329/337] Add files via upload --- 7.0/models/RL/tensorflow/non_parallel/DDPG.py | 2 +- 7.0/models/RL/tensorflow/non_parallel/PPO.py | 2 +- 7.0/models/RL/tensorflow/non_parallel/PPO_eps.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/7.0/models/RL/tensorflow/non_parallel/DDPG.py b/7.0/models/RL/tensorflow/non_parallel/DDPG.py index 6474489a..83a84db6 100644 --- a/7.0/models/RL/tensorflow/non_parallel/DDPG.py +++ b/7.0/models/RL/tensorflow/non_parallel/DDPG.py @@ -65,7 +65,7 @@ def loss(self,s,a,next_s,r,d): # loss function, kernel uses it to calculate los q_target=tf.cast(r,'float32')+self.gamma*next_q_value*(1-tf.cast(d,'float32')) # calculate the target value using Bellman equation with discount factor gamma actor_loss=-tf.reduce_mean(self.critic.fp(s,self.actor.fp(s))) # calculate the actor loss as the negative mean of the Q-value for the current state and action from the critic network critic_loss=tf.reduce_mean((self.critic.fp(s,a)-q_target)**2) # calculate the critic loss as the mean squared error between Q-value and target value - return [actor_loss,critic_loss] # return a list of actor loss and critic loss + return actor_loss+critic_loss # return a sum of actor loss and critic loss def update_param(self): # update function, kernel uses it to update parameter diff --git a/7.0/models/RL/tensorflow/non_parallel/PPO.py b/7.0/models/RL/tensorflow/non_parallel/PPO.py index dbce55f2..c0e9b704 100644 --- a/7.0/models/RL/tensorflow/non_parallel/PPO.py +++ b/7.0/models/RL/tensorflow/non_parallel/PPO.py @@ -60,7 +60,7 @@ def loss(self,s,a,next_s,r,d): clip_loss=-tf.math.minimum(sur1,sur2) entropy=action_prob*tf.math.log(action_prob+1e-8) clip_loss=clip_loss-self.alpha*entropy - return [tf.reduce_mean(clip_loss),tf.reduce_mean((TD)**2)] + return tf.reduce_mean(clip_loss)+tf.reduce_mean((TD)**2) def update_param(self): diff --git a/7.0/models/RL/tensorflow/non_parallel/PPO_eps.py b/7.0/models/RL/tensorflow/non_parallel/PPO_eps.py index 899eccae..1287521b 100644 --- a/7.0/models/RL/tensorflow/non_parallel/PPO_eps.py +++ b/7.0/models/RL/tensorflow/non_parallel/PPO_eps.py @@ -55,7 +55,7 @@ def loss(self,s,a,next_s,r,d): sur1=raito*TD sur2=tf.clip_by_value(raito,clip_value_min=1-self.clip_eps,clip_value_max=1+self.clip_eps)*TD clip_loss=-tf.math.minimum(sur1,sur2) - return [tf.reduce_mean(clip_loss),tf.reduce_mean((TD)**2)] + return tf.reduce_mean(clip_loss)+tf.reduce_mean((TD)**2) def update_param(self): From 14ea52ccd60aa49f3b9c905de4406e9195b7a815 Mon Sep 17 00:00:00 2001 From: NoteDance Date: Thu, 13 Feb 2025 18:35:26 +0800 Subject: [PATCH 330/337] Add files via upload --- 7.0/models/RL/tensorflow/parallel/DDPG.py | 2 +- 7.0/models/RL/tensorflow/parallel/PPO.py | 2 +- 7.0/models/RL/tensorflow/parallel/PPO_eps.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/7.0/models/RL/tensorflow/parallel/DDPG.py b/7.0/models/RL/tensorflow/parallel/DDPG.py index 2b2f7d99..d8d9b483 100644 --- a/7.0/models/RL/tensorflow/parallel/DDPG.py +++ b/7.0/models/RL/tensorflow/parallel/DDPG.py @@ -66,7 +66,7 @@ def loss(self,s,a,next_s,r,d): # loss function, kernel uses it to calculate los q_target=tf.cast(r,'float32')+self.gamma*next_q_value*(1-tf.cast(d,'float32')) # calculate the target value using Bellman equation with discount factor gamma actor_loss=-tf.reduce_mean(self.critic.fp(s,self.actor.fp(s))) # calculate the actor loss as the negative mean of the Q-value for the current state and action from the critic network critic_loss=tf.reduce_mean((self.critic.fp(s,a)-q_target)**2) # calculate the critic loss as the mean squared error between Q-value and target value - return [actor_loss,critic_loss] # return a list of actor loss and critic loss + return actor_loss+critic_loss # return a sum of actor loss and critic loss def update_param(self): # update function, kernel uses it to update parameter diff --git a/7.0/models/RL/tensorflow/parallel/PPO.py b/7.0/models/RL/tensorflow/parallel/PPO.py index dbf28400..88763150 100644 --- a/7.0/models/RL/tensorflow/parallel/PPO.py +++ b/7.0/models/RL/tensorflow/parallel/PPO.py @@ -60,7 +60,7 @@ def loss(self,s,a,next_s,r,d): clip_loss=-tf.math.minimum(sur1,sur2) entropy=action_prob*tf.math.log(action_prob+1e-8) clip_loss=clip_loss-self.alpha*entropy - return [tf.reduce_mean(clip_loss),tf.reduce_mean((TD)**2)] + return tf.reduce_mean(clip_loss)+tf.reduce_mean((TD)**2) def update_param(self): diff --git a/7.0/models/RL/tensorflow/parallel/PPO_eps.py b/7.0/models/RL/tensorflow/parallel/PPO_eps.py index 4a4eb6af..008baf04 100644 --- a/7.0/models/RL/tensorflow/parallel/PPO_eps.py +++ b/7.0/models/RL/tensorflow/parallel/PPO_eps.py @@ -56,7 +56,7 @@ def loss(self,s,a,next_s,r,d): sur1=raito*TD sur2=tf.clip_by_value(raito,clip_value_min=1-self.clip_eps,clip_value_max=1+self.clip_eps)*TD clip_loss=-tf.math.minimum(sur1,sur2) - return [tf.reduce_mean(clip_loss),tf.reduce_mean((TD)**2)] + return tf.reduce_mean(clip_loss)+tf.reduce_mean((TD)**2) def update_param(self): From e69ba9ff866e0f32cf830662977ddd5abaf78151 Mon Sep 17 00:00:00 2001 From: NoteDance Date: Sat, 22 Feb 2025 00:18:28 +0800 Subject: [PATCH 331/337] Add files via upload --- 7.0/models/RL/tensorflow/non_parallel/DDPG.py | 10 +++++----- 7.0/models/RL/tensorflow/non_parallel/DQN.py | 6 +++--- 7.0/models/RL/tensorflow/non_parallel/DQN_pr.py | 8 ++++---- 7.0/models/RL/tensorflow/non_parallel/PPO.py | 12 ++++++------ 7.0/models/RL/tensorflow/non_parallel/PPO_eps.py | 10 +++++----- 5 files changed, 23 insertions(+), 23 deletions(-) diff --git a/7.0/models/RL/tensorflow/non_parallel/DDPG.py b/7.0/models/RL/tensorflow/non_parallel/DDPG.py index 83a84db6..41cb8fd3 100644 --- a/7.0/models/RL/tensorflow/non_parallel/DDPG.py +++ b/7.0/models/RL/tensorflow/non_parallel/DDPG.py @@ -11,7 +11,7 @@ def __init__(self,state_dim,hidden_dim,action_dim,action_bound): # initialize th self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list - def fp(self,x): # forward propagation function, kernel uses it for forward propagation + def __call__(self,x): # forward propagation function, kernel uses it for forward propagation x = self.dense1(x) return self.dense2(x)*self.action_bound @@ -23,7 +23,7 @@ def __init__(self,state_dim,hidden_dim,action_dim): # initialize the network wit self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list - def fp(self,x,a): # forward propagation function, kernel uses it for forward propagation + def __call__(self,x,a): # forward propagation function, kernel uses it for forward propagation cat=tf.concat([x,a],axis=1) x=self.dense1(cat) return self.dense2(x) @@ -61,10 +61,10 @@ def env(self,a=None,initial=None): # environment function, kernel uses it to in def loss(self,s,a,next_s,r,d): # loss function, kernel uses it to calculate loss a=tf.expand_dims(a,axis=1) # expand the action vector to match the Q-value matrix shape - next_q_value=self.target_critic.fp(next_s,self.target_actor.fp(next_s)) # get the Q-value for the next state and action from the target critic network + next_q_value=self.target_critic(next_s,self.target_actor(next_s)) # get the Q-value for the next state and action from the target critic network q_target=tf.cast(r,'float32')+self.gamma*next_q_value*(1-tf.cast(d,'float32')) # calculate the target value using Bellman equation with discount factor gamma - actor_loss=-tf.reduce_mean(self.critic.fp(s,self.actor.fp(s))) # calculate the actor loss as the negative mean of the Q-value for the current state and action from the critic network - critic_loss=tf.reduce_mean((self.critic.fp(s,a)-q_target)**2) # calculate the critic loss as the mean squared error between Q-value and target value + actor_loss=-tf.reduce_mean(self.critic(s,self.actor(s))) # calculate the actor loss as the negative mean of the Q-value for the current state and action from the critic network + critic_loss=tf.reduce_mean((self.critic(s,a)-q_target)**2) # calculate the critic loss as the mean squared error between Q-value and target value return actor_loss+critic_loss # return a sum of actor loss and critic loss diff --git a/7.0/models/RL/tensorflow/non_parallel/DQN.py b/7.0/models/RL/tensorflow/non_parallel/DQN.py index d0be2840..aa897704 100644 --- a/7.0/models/RL/tensorflow/non_parallel/DQN.py +++ b/7.0/models/RL/tensorflow/non_parallel/DQN.py @@ -9,7 +9,7 @@ def __init__(self,state_dim,hidden_dim,action_dim): # initialize the network wit self.dense2 = nn.dense(action_dim, hidden_dim) self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list - def fp(self,x): # forward propagation function, kernel uses it for forward propagation + def __call__(self,x): # forward propagation function, kernel uses it for forward propagation x = self.dense2(self.dense1(x)) return x @@ -34,8 +34,8 @@ def env(self,a=None,initial=None): # environment function, kernel uses it to int def loss(self,s,a,next_s,r,d): # loss function, kernel uses it to calculate loss a=tf.expand_dims(a,axis=1) # expand the action vector to match the Q-value matrix shape - q_value=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) # get the Q-value for the selected action - next_q_value=tf.reduce_max(self.target_q_net.fp(next_s),axis=1) # get the maximum Q-value for the next state from the target network + q_value=tf.gather(self.nn(s),a,axis=1,batch_dims=1) # get the Q-value for the selected action + next_q_value=tf.reduce_max(self.target_q_net(next_s),axis=1) # get the maximum Q-value for the next state from the target network target=tf.cast(r,'float32')+0.98*next_q_value*(1-tf.cast(d,'float32')) # calculate the target value using Bellman equation with discount factor 0.98 return tf.reduce_mean((q_value-target)**2) # return the mean squared error between Q-value and target value diff --git a/7.0/models/RL/tensorflow/non_parallel/DQN_pr.py b/7.0/models/RL/tensorflow/non_parallel/DQN_pr.py index c9832b1d..5d105c3c 100644 --- a/7.0/models/RL/tensorflow/non_parallel/DQN_pr.py +++ b/7.0/models/RL/tensorflow/non_parallel/DQN_pr.py @@ -10,7 +10,7 @@ def __init__(self,state_dim,hidden_dim,action_dim): self.dense2 = nn.dense(action_dim, hidden_dim) self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list - def fp(self,x): + def __call__(self,x): x = self.dense2(self.dense1(x)) return x @@ -44,8 +44,8 @@ def data_func(self,state_pool,action_pool,next_state_pool,reward_pool,done_pool, def loss(self,s,a,next_s,r,d): a=tf.expand_dims(a,axis=1) - q_value=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) - next_q_value=tf.reduce_max(self.target_q_net.fp(next_s),axis=1) + q_value=tf.gather(self.nn(s),a,axis=1,batch_dims=1) + next_q_value=tf.reduce_max(self.target_q_net(next_s),axis=1) target=tf.cast(r,'float32')+0.98*next_q_value*(1-tf.cast(d,'float32')) target=tf.expand_dims(target,axis=1) TD=target-q_value @@ -55,4 +55,4 @@ def loss(self,s,a,next_s,r,d): def update_param(self): nn.assign_param(self.target_q_net.param, self.param) - return + return \ No newline at end of file diff --git a/7.0/models/RL/tensorflow/non_parallel/PPO.py b/7.0/models/RL/tensorflow/non_parallel/PPO.py index c0e9b704..58707b1d 100644 --- a/7.0/models/RL/tensorflow/non_parallel/PPO.py +++ b/7.0/models/RL/tensorflow/non_parallel/PPO.py @@ -9,7 +9,7 @@ def __init__(self,state_dim,hidden_dim,action_dim): self.dense2 = nn.dense(action_dim, hidden_dim) self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list - def fp(self,x): + def __call__(self,x): x=self.dense1(x) return tf.nn.softmax(self.dense2(x)) @@ -20,7 +20,7 @@ def __init__(self,state_dim,hidden_dim): self.dense2 = nn.dense(1, hidden_dim) self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list - def fp(self,x): + def __call__(self,x): x=self.dense1(x) return self.dense2(x) @@ -49,11 +49,11 @@ def env(self,a=None,initial=None): def loss(self,s,a,next_s,r,d): a=tf.expand_dims(a,axis=1) - action_prob=tf.gather(self.actor.fp(s),a,axis=1,batch_dims=1) - action_prob_old=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) + action_prob=tf.gather(self.actor(s),a,axis=1,batch_dims=1) + action_prob_old=tf.gather(self.nn(s),a,axis=1,batch_dims=1) raito=action_prob/action_prob_old - value=self.critic.fp(s) - value_tar=tf.cast(r,'float32')+0.98*self.critic.fp(next_s)*(1-tf.cast(d,'float32')) + value=self.critic(s) + value_tar=tf.cast(r,'float32')+0.98*self.critic(next_s)*(1-tf.cast(d,'float32')) TD=value_tar-value sur1=raito*TD sur2=tf.clip_by_value(raito,clip_value_min=1-self.clip_eps,clip_value_max=1+self.clip_eps)*TD diff --git a/7.0/models/RL/tensorflow/non_parallel/PPO_eps.py b/7.0/models/RL/tensorflow/non_parallel/PPO_eps.py index 1287521b..14cabe9a 100644 --- a/7.0/models/RL/tensorflow/non_parallel/PPO_eps.py +++ b/7.0/models/RL/tensorflow/non_parallel/PPO_eps.py @@ -9,7 +9,7 @@ def __init__(self,state_dim,hidden_dim,action_dim): self.dense2 = nn.dense(action_dim, hidden_dim) self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list - def fp(self,x): + def __call__(self,x): x=self.dense1(x) return tf.nn.softmax(self.dense2(x)) @@ -20,7 +20,7 @@ def __init__(self,state_dim,hidden_dim): self.dense2 = nn.dense(1, hidden_dim) self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list - def fp(self,x): + def __call__(self,x): x=self.dense1(x) return self.dense2(x) @@ -48,9 +48,9 @@ def env(self,a=None,initial=None): def loss(self,s,a,next_s,r,d): a=tf.expand_dims(a,axis=1) - raito=tf.gather(self.actor.fp(s),a,axis=1,batch_dims=1)/tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) - value=self.critic.fp(s) - value_tar=tf.cast(r,'float32')+0.98*self.critic.fp(next_s)*(1-tf.cast(d,'float32')) + raito=tf.gather(self.actor(s),a,axis=1,batch_dims=1)/tf.gather(self.nn(s),a,axis=1,batch_dims=1) + value=self.critic(s) + value_tar=tf.cast(r,'float32')+0.98*self.critic(next_s)*(1-tf.cast(d,'float32')) TD=value_tar-value sur1=raito*TD sur2=tf.clip_by_value(raito,clip_value_min=1-self.clip_eps,clip_value_max=1+self.clip_eps)*TD From fdd936dde54c0c530d8befd6594cfa42cbd7d045 Mon Sep 17 00:00:00 2001 From: NoteDance Date: Sat, 22 Feb 2025 00:19:22 +0800 Subject: [PATCH 332/337] Add files via upload --- 7.0/models/RL/tensorflow/parallel/DDPG.py | 10 +++++----- 7.0/models/RL/tensorflow/parallel/DQN.py | 6 +++--- 7.0/models/RL/tensorflow/parallel/DQN_pr.py | 8 ++++---- 7.0/models/RL/tensorflow/parallel/PPO.py | 12 ++++++------ 7.0/models/RL/tensorflow/parallel/PPO_eps.py | 10 +++++----- 5 files changed, 23 insertions(+), 23 deletions(-) diff --git a/7.0/models/RL/tensorflow/parallel/DDPG.py b/7.0/models/RL/tensorflow/parallel/DDPG.py index d8d9b483..e4068c49 100644 --- a/7.0/models/RL/tensorflow/parallel/DDPG.py +++ b/7.0/models/RL/tensorflow/parallel/DDPG.py @@ -12,7 +12,7 @@ def __init__(self,state_dim,hidden_dim,action_dim,action_bound): # initialize th self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list - def fp(self,x): # forward propagation function, kernel uses it for forward propagation + def __call__(self,x): # forward propagation function, kernel uses it for forward propagation x = self.dense1(x) return self.dense2(x)*self.action_bound @@ -24,7 +24,7 @@ def __init__(self,state_dim,hidden_dim,action_dim): # initialize the network wit self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list - def fp(self,x,a): # forward propagation function, kernel uses it for forward propagation + def __call__(self,x,a): # forward propagation function, kernel uses it for forward propagation cat=tf.concat([x,a],axis=1) x=self.dense1(cat) return self.dense2(x) @@ -62,10 +62,10 @@ def env(self,a=None,p=None,initial=None): # environment function, kernel uses it def loss(self,s,a,next_s,r,d): # loss function, kernel uses it to calculate loss a=tf.expand_dims(a,axis=1) # expand the action vector to match the Q-value matrix shape - next_q_value=self.target_critic.fp(next_s,self.target_actor.fp(next_s)) # get the Q-value for the next state and action from the target critic network + next_q_value=self.target_critic(next_s,self.target_actor(next_s)) # get the Q-value for the next state and action from the target critic network q_target=tf.cast(r,'float32')+self.gamma*next_q_value*(1-tf.cast(d,'float32')) # calculate the target value using Bellman equation with discount factor gamma - actor_loss=-tf.reduce_mean(self.critic.fp(s,self.actor.fp(s))) # calculate the actor loss as the negative mean of the Q-value for the current state and action from the critic network - critic_loss=tf.reduce_mean((self.critic.fp(s,a)-q_target)**2) # calculate the critic loss as the mean squared error between Q-value and target value + actor_loss=-tf.reduce_mean(self.critic(s,self.actor(s))) # calculate the actor loss as the negative mean of the Q-value for the current state and action from the critic network + critic_loss=tf.reduce_mean((self.critic(s,a)-q_target)**2) # calculate the critic loss as the mean squared error between Q-value and target value return actor_loss+critic_loss # return a sum of actor loss and critic loss diff --git a/7.0/models/RL/tensorflow/parallel/DQN.py b/7.0/models/RL/tensorflow/parallel/DQN.py index 9186307d..6cf52803 100644 --- a/7.0/models/RL/tensorflow/parallel/DQN.py +++ b/7.0/models/RL/tensorflow/parallel/DQN.py @@ -10,7 +10,7 @@ def __init__(self,state_dim,hidden_dim,action_dim): # initialize the network wit self.dense2 = nn.dense(action_dim, hidden_dim) self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list - def fp(self,x): # forward propagation function, kernel uses it for forward propagation + def __call__(self,x): # forward propagation function, kernel uses it for forward propagation x = self.dense2(self.dense1(x)) return x @@ -35,8 +35,8 @@ def env(self,a=None,p=None,initial=None): # environment function, kernel uses it def loss(self,s,a,next_s,r,d): # loss function, kernel uses it to calculate loss a=tf.expand_dims(a,axis=1) # expand the action vector to match the Q-value matrix shape - q_value=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) # get the Q-value for the selected action - next_q_value=tf.reduce_max(self.target_q_net.fp(next_s),axis=1) # get the maximum Q-value for the next state from the target network + q_value=tf.gather(self.nn(s),a,axis=1,batch_dims=1) # get the Q-value for the selected action + next_q_value=tf.reduce_max(self.target_q_net(next_s),axis=1) # get the maximum Q-value for the next state from the target network target=tf.cast(r,'float32')+0.98*next_q_value*(1-tf.cast(d,'float32')) # calculate the target value using Bellman equation with discount factor 0.98 return tf.reduce_mean((q_value-target)**2) # return the mean squared error between Q-value and target value diff --git a/7.0/models/RL/tensorflow/parallel/DQN_pr.py b/7.0/models/RL/tensorflow/parallel/DQN_pr.py index 16192390..99b956a0 100644 --- a/7.0/models/RL/tensorflow/parallel/DQN_pr.py +++ b/7.0/models/RL/tensorflow/parallel/DQN_pr.py @@ -11,7 +11,7 @@ def __init__(self,state_dim,hidden_dim,action_dim): # initialize the network wit self.dense2 = nn.dense(action_dim, hidden_dim) self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list - def fp(self,x): # forward propagation function, kernel uses it for forward propagation + def __call__(self,x): # forward propagation function, kernel uses it for forward propagation x = self.dense2(self.dense1(x)) return x @@ -45,8 +45,8 @@ def data_func(self,state_pool,action_pool,next_state_pool,reward_pool,done_pool, def loss(self,s,a,next_s,r,d): # loss function, kernel uses it to calculate loss a=tf.expand_dims(a,axis=1) # expand the action vector to match the Q-value matrix shape - q_value=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) # get the Q-value for the selected action - next_q_value=tf.reduce_max(self.target_q_net.fp(next_s),axis=1) # get the maximum Q-value for the next state from the target network + q_value=tf.gather(self.nn(s),a,axis=1,batch_dims=1) # get the Q-value for the selected action + next_q_value=tf.reduce_max(self.target_q_net(next_s),axis=1) # get the maximum Q-value for the next state from the target network target=tf.cast(r,'float32')+0.98*next_q_value*(1-tf.cast(d,'float32')) # calculate the target value using Bellman equation with discount factor 0.98 TD=q_value-target self.pr.update_TD(TD) @@ -60,4 +60,4 @@ def update_param(self): # update function, kernel uses it to update parameter def opt(self,gradient): # optimization function, kernel uses it to optimize parameter param=self.optimizer.opt(gradient,self.param) # apply the custom momentum optimizer to update the parameters using the gradient - return param + return param \ No newline at end of file diff --git a/7.0/models/RL/tensorflow/parallel/PPO.py b/7.0/models/RL/tensorflow/parallel/PPO.py index 88763150..8b162420 100644 --- a/7.0/models/RL/tensorflow/parallel/PPO.py +++ b/7.0/models/RL/tensorflow/parallel/PPO.py @@ -10,7 +10,7 @@ def __init__(self,state_dim,hidden_dim,action_dim): self.dense2 = nn.dense(action_dim, hidden_dim) self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list - def fp(self,x): + def __call__(self,x): x=self.dense1(x) return tf.nn.softmax(self.dense2(x)) @@ -21,7 +21,7 @@ def __init__(self,state_dim,hidden_dim): self.dense2 = nn.dense(1, hidden_dim) self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list - def fp(self,x): + def __call__(self,x): x=self.dense1(x) return self.dense2(x) @@ -49,11 +49,11 @@ def env(self,a=None,p=None,initial=None): def loss(self,s,a,next_s,r,d): a=tf.expand_dims(a,axis=1) - action_prob=tf.gather(self.actor.fp(s),a,axis=1,batch_dims=1) - action_prob_old=tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) + action_prob=tf.gather(self.actor(s),a,axis=1,batch_dims=1) + action_prob_old=tf.gather(self.nn(s),a,axis=1,batch_dims=1) raito=action_prob/action_prob_old - value=self.critic.fp(s) - value_tar=tf.cast(r,'float32')+0.98*self.critic.fp(next_s)*(1-tf.cast(d,'float32')) + value=self.critic(s) + value_tar=tf.cast(r,'float32')+0.98*self.critic(next_s)*(1-tf.cast(d,'float32')) TD=value_tar-value sur1=raito*TD sur2=tf.clip_by_value(raito,clip_value_min=1-self.clip_eps,clip_value_max=1+self.clip_eps)*TD diff --git a/7.0/models/RL/tensorflow/parallel/PPO_eps.py b/7.0/models/RL/tensorflow/parallel/PPO_eps.py index 008baf04..3e21419d 100644 --- a/7.0/models/RL/tensorflow/parallel/PPO_eps.py +++ b/7.0/models/RL/tensorflow/parallel/PPO_eps.py @@ -10,7 +10,7 @@ def __init__(self,state_dim,hidden_dim,action_dim): self.dense2 = nn.dense(action_dim, hidden_dim) self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list - def fp(self,x): + def __call__(self,x): x=self.dense1(x) return tf.nn.softmax(self.dense2(x)) @@ -21,7 +21,7 @@ def __init__(self,state_dim,hidden_dim): self.dense2 = nn.dense(1, hidden_dim) self.param=[self.dense1.param,self.dense2.param] # store the network parameters in a list - def fp(self,x): + def __call__(self,x): x=self.dense1(x) return self.dense2(x) @@ -49,9 +49,9 @@ def env(self,a=None,p=None,initial=None): def loss(self,s,a,next_s,r,d): a=tf.expand_dims(a,axis=1) - raito=tf.gather(self.actor.fp(s),a,axis=1,batch_dims=1)/tf.gather(self.nn.fp(s),a,axis=1,batch_dims=1) - value=self.critic.fp(s) - value_tar=tf.cast(r,'float32')+0.98*self.critic.fp(next_s)*(1-tf.cast(d,'float32')) + raito=tf.gather(self.actor(s),a,axis=1,batch_dims=1)/tf.gather(self.nn(s),a,axis=1,batch_dims=1) + value=self.critic(s) + value_tar=tf.cast(r,'float32')+0.98*self.critic(next_s)*(1-tf.cast(d,'float32')) TD=value_tar-value sur1=raito*TD sur2=tf.clip_by_value(raito,clip_value_min=1-self.clip_eps,clip_value_max=1+self.clip_eps)*TD From 3b86f110187d4ee0422e2a43636a6efbbd2b2260 Mon Sep 17 00:00:00 2001 From: NoteDance Date: Sat, 22 Feb 2025 12:59:51 +0800 Subject: [PATCH 333/337] Add files via upload --- 7.0/models/RL/tensorflow/parallel/DDPG.py | 10 ++++++++-- 7.0/models/RL/tensorflow/parallel/DQN.py | 4 ++-- 7.0/models/RL/tensorflow/parallel/DQN_pr.py | 4 ++-- 7.0/models/RL/tensorflow/parallel/PPO.py | 12 +++++++++--- 7.0/models/RL/tensorflow/parallel/PPO_eps.py | 12 +++++++++--- 5 files changed, 30 insertions(+), 12 deletions(-) diff --git a/7.0/models/RL/tensorflow/parallel/DDPG.py b/7.0/models/RL/tensorflow/parallel/DDPG.py index e4068c49..821a9c4a 100644 --- a/7.0/models/RL/tensorflow/parallel/DDPG.py +++ b/7.0/models/RL/tensorflow/parallel/DDPG.py @@ -48,7 +48,7 @@ def __init__(self,hidden_dim,sigma,gamma,tau,actor_lr,critic_lr): # initialize t self.sigma=sigma # noise scale self.gamma=gamma # discount factor self.tau=tau # soft update factor - self.optimizer=o.SGD(param=self.param) # optimizer, kernel uses it to optimize. Here we use a custom SGD optimizer + self.optimizer=[o.SGD(param=self.param[0]),o.SGD(param=self.param[1])] # optimizer, kernel uses it to optimize. Here we use a custom SGD optimizer def env(self,a=None,p=None,initial=None): # environment function, kernel uses it to interact with the environment @@ -74,4 +74,10 @@ def update_param(self): # update function, kernel uses it to update parameter target_param.assign(target_param*(1.0-self.tau)+param*self.tau) # update the target parameter using soft update with factor tau for target_param,param in zip(self.target_critic.param,self.critic.param): # for each pair of parameters in target critic network and critic network target_param.assign(target_param*(1.0-self.tau)+param*self.tau) # update the target parameter using soft update with factor tau - return \ No newline at end of file + return + + + def opt(self,gradient): # optimization function, kernel uses it to optimize parameter + self.optimizer[0](gradient,self.param[0]) # apply the custom momentum optimizer to update the parameters using the gradient + self.optimizer[1](gradient,self.param[1]) + return self.param \ No newline at end of file diff --git a/7.0/models/RL/tensorflow/parallel/DQN.py b/7.0/models/RL/tensorflow/parallel/DQN.py index 6cf52803..ad7d3738 100644 --- a/7.0/models/RL/tensorflow/parallel/DQN.py +++ b/7.0/models/RL/tensorflow/parallel/DQN.py @@ -47,5 +47,5 @@ def update_param(self): # update function, kernel uses it to update parameter def opt(self,gradient): # optimization function, kernel uses it to optimize parameter - param=self.optimizer.opt(gradient,self.param) # apply the custom momentum optimizer to update the parameters using the gradient - return param \ No newline at end of file + self.optimizer(gradient,self.param) # apply the custom momentum optimizer to update the parameters using the gradient + return self.param \ No newline at end of file diff --git a/7.0/models/RL/tensorflow/parallel/DQN_pr.py b/7.0/models/RL/tensorflow/parallel/DQN_pr.py index 99b956a0..b4600a0f 100644 --- a/7.0/models/RL/tensorflow/parallel/DQN_pr.py +++ b/7.0/models/RL/tensorflow/parallel/DQN_pr.py @@ -59,5 +59,5 @@ def update_param(self): # update function, kernel uses it to update parameter def opt(self,gradient): # optimization function, kernel uses it to optimize parameter - param=self.optimizer.opt(gradient,self.param) # apply the custom momentum optimizer to update the parameters using the gradient - return param \ No newline at end of file + self.optimizer(gradient,self.param) # apply the custom momentum optimizer to update the parameters using the gradient + return self.param \ No newline at end of file diff --git a/7.0/models/RL/tensorflow/parallel/PPO.py b/7.0/models/RL/tensorflow/parallel/PPO.py index 8b162420..120bb845 100644 --- a/7.0/models/RL/tensorflow/parallel/PPO.py +++ b/7.0/models/RL/tensorflow/parallel/PPO.py @@ -1,7 +1,7 @@ import tensorflow as tf from Note import nn import gym -from Note.nn.parallel.optimizer import SGD +import Note.nn.parallel.optimizer as o class actor: @@ -34,7 +34,7 @@ def __init__(self,state_dim,hidden_dim,action_dim,clip_eps,alpha): self.critic=critic(state_dim,hidden_dim) self.clip_eps=clip_eps self.param=[self.actor.param,self.critic.param] - self.opt=SGD(param=self.param) + self.opt=[o.SGD(param=self.param[0]),o.SGD(param=self.param[1])] self.genv=[gym.make('CartPole-v0') for _ in range(5)] @@ -65,4 +65,10 @@ def loss(self,s,a,next_s,r,d): def update_param(self): nn.assign(self.nn.param,self.actor.param) - return \ No newline at end of file + return + + + def opt(self,gradient): + self.optimizer[0](gradient,self.param[0]) + self.optimizer[1](gradient,self.param[1]) + return self.param \ No newline at end of file diff --git a/7.0/models/RL/tensorflow/parallel/PPO_eps.py b/7.0/models/RL/tensorflow/parallel/PPO_eps.py index 3e21419d..4c11210a 100644 --- a/7.0/models/RL/tensorflow/parallel/PPO_eps.py +++ b/7.0/models/RL/tensorflow/parallel/PPO_eps.py @@ -1,7 +1,7 @@ import tensorflow as tf from Note import nn import gym -from Note.nn.parallel.optimizer import SGD +import Note.nn.parallel.optimizer as o class actor: @@ -34,7 +34,7 @@ def __init__(self,state_dim,hidden_dim,action_dim,clip_eps): self.critic=critic(state_dim,hidden_dim) self.clip_eps=clip_eps self.param=[self.actor.param,self.critic.param] - self.opt=SGD(param=self.param) + self.opt=[o.SGD(param=self.param[0]),o.SGD(param=self.param[1])] self.genv=[gym.make('CartPole-v0') for _ in range(5)] @@ -61,4 +61,10 @@ def loss(self,s,a,next_s,r,d): def update_param(self): nn.assign(self.nn.param,self.actor.param) - return \ No newline at end of file + return + + + def opt(self,gradient): + self.optimizer[0](gradient,self.param[0]) + self.optimizer[1](gradient,self.param[1]) + return self.param \ No newline at end of file From c8932bedc94768e178e53be8a6a04937672b6a5c Mon Sep 17 00:00:00 2001 From: NoteDance Date: Sat, 22 Feb 2025 17:22:59 +0800 Subject: [PATCH 334/337] Add files via upload --- 7.0/models/RL/tensorflow/non_parallel/PPO.py | 12 ++++++------ 7.0/models/RL/tensorflow/non_parallel/PPO_eps.py | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/7.0/models/RL/tensorflow/non_parallel/PPO.py b/7.0/models/RL/tensorflow/non_parallel/PPO.py index 58707b1d..cb1cbd55 100644 --- a/7.0/models/RL/tensorflow/non_parallel/PPO.py +++ b/7.0/models/RL/tensorflow/non_parallel/PPO.py @@ -27,14 +27,14 @@ def __call__(self,x): class PPO: def __init__(self,state_dim,hidden_dim,action_dim,clip_eps,alpha): + self.actor_=actor(state_dim,hidden_dim,action_dim) self.actor=actor(state_dim,hidden_dim,action_dim) - self.nn=actor(state_dim,hidden_dim,action_dim) - nn.assign_param(self.nn.param,self.actor.param.copy()) + nn.assign_param(self.actor.param,self.actor_.param) self.critic=critic(state_dim,hidden_dim) self.clip_eps=clip_eps self.alpha=alpha - self.param=[self.actor.param,self.critic.param] - self.opt=tf.keras.optimizers.Adam() + self.param=[self.actor_.param,self.critic.param] + self.opt=[tf.keras.optimizers.Adam(),tf.keras.optimizers.Adam()] self.genv=gym.make('CartPole-v0') @@ -49,8 +49,8 @@ def env(self,a=None,initial=None): def loss(self,s,a,next_s,r,d): a=tf.expand_dims(a,axis=1) - action_prob=tf.gather(self.actor(s),a,axis=1,batch_dims=1) - action_prob_old=tf.gather(self.nn(s),a,axis=1,batch_dims=1) + action_prob=tf.gather(self.actor_(s),a,axis=1,batch_dims=1) + action_prob_old=tf.gather(self.actor(s),a,axis=1,batch_dims=1) raito=action_prob/action_prob_old value=self.critic(s) value_tar=tf.cast(r,'float32')+0.98*self.critic(next_s)*(1-tf.cast(d,'float32')) diff --git a/7.0/models/RL/tensorflow/non_parallel/PPO_eps.py b/7.0/models/RL/tensorflow/non_parallel/PPO_eps.py index 14cabe9a..6b2eeb98 100644 --- a/7.0/models/RL/tensorflow/non_parallel/PPO_eps.py +++ b/7.0/models/RL/tensorflow/non_parallel/PPO_eps.py @@ -27,13 +27,13 @@ def __call__(self,x): class PPO: def __init__(self,state_dim,hidden_dim,action_dim,clip_eps): + self.actor_=actor(state_dim,hidden_dim,action_dim) self.actor=actor(state_dim,hidden_dim,action_dim) - self.nn=actor(state_dim,hidden_dim,action_dim) - nn.assign_param(self.nn.param,self.actor.param.copy()) + nn.assign_param(self.actor.param,self.actor_.param) self.critic=critic(state_dim,hidden_dim) self.clip_eps=clip_eps - self.param=[self.actor.param,self.critic.param] - self.opt=tf.keras.optimizers.Adam() + self.param=[self.actor_.param,self.critic.param] + self.opt=[tf.keras.optimizers.Adam(),tf.keras.optimizers.Adam()] self.genv=gym.make('CartPole-v0') @@ -48,7 +48,7 @@ def env(self,a=None,initial=None): def loss(self,s,a,next_s,r,d): a=tf.expand_dims(a,axis=1) - raito=tf.gather(self.actor(s),a,axis=1,batch_dims=1)/tf.gather(self.nn(s),a,axis=1,batch_dims=1) + raito=tf.gather(self.actor_(s),a,axis=1,batch_dims=1)/tf.gather(self.actor(s),a,axis=1,batch_dims=1) value=self.critic(s) value_tar=tf.cast(r,'float32')+0.98*self.critic(next_s)*(1-tf.cast(d,'float32')) TD=value_tar-value @@ -59,5 +59,5 @@ def loss(self,s,a,next_s,r,d): def update_param(self): - nn.assign_param(self.nn.param, self.actor.param) + nn.assign_param(self.actor.param, self.actor_.param) return \ No newline at end of file From b8174a2e1064e49d3c3b3fbaa2e7fe7d1e00a9bd Mon Sep 17 00:00:00 2001 From: NoteDance Date: Sat, 22 Feb 2025 17:23:24 +0800 Subject: [PATCH 335/337] Add files via upload --- 7.0/models/RL/tensorflow/parallel/PPO.py | 12 ++++++------ 7.0/models/RL/tensorflow/parallel/PPO_eps.py | 10 +++++----- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/7.0/models/RL/tensorflow/parallel/PPO.py b/7.0/models/RL/tensorflow/parallel/PPO.py index 120bb845..dcda703a 100644 --- a/7.0/models/RL/tensorflow/parallel/PPO.py +++ b/7.0/models/RL/tensorflow/parallel/PPO.py @@ -28,12 +28,12 @@ def __call__(self,x): class PPO: def __init__(self,state_dim,hidden_dim,action_dim,clip_eps,alpha): + self.actor_=actor(state_dim,hidden_dim,action_dim) self.actor=actor(state_dim,hidden_dim,action_dim) - self.nn=actor(state_dim,hidden_dim,action_dim) - nn.assign(self.nn.param,self.actor.param.copy()) + nn.assign(self.actor.param,self.actor_.param) self.critic=critic(state_dim,hidden_dim) self.clip_eps=clip_eps - self.param=[self.actor.param,self.critic.param] + self.param=[self.actor_.param,self.critic.param] self.opt=[o.SGD(param=self.param[0]),o.SGD(param=self.param[1])] self.genv=[gym.make('CartPole-v0') for _ in range(5)] @@ -49,8 +49,8 @@ def env(self,a=None,p=None,initial=None): def loss(self,s,a,next_s,r,d): a=tf.expand_dims(a,axis=1) - action_prob=tf.gather(self.actor(s),a,axis=1,batch_dims=1) - action_prob_old=tf.gather(self.nn(s),a,axis=1,batch_dims=1) + action_prob=tf.gather(self.actor_(s),a,axis=1,batch_dims=1) + action_prob_old=tf.gather(self.actor(s),a,axis=1,batch_dims=1) raito=action_prob/action_prob_old value=self.critic(s) value_tar=tf.cast(r,'float32')+0.98*self.critic(next_s)*(1-tf.cast(d,'float32')) @@ -64,7 +64,7 @@ def loss(self,s,a,next_s,r,d): def update_param(self): - nn.assign(self.nn.param,self.actor.param) + nn.assign(self.actor.param,self.actor_.param) return diff --git a/7.0/models/RL/tensorflow/parallel/PPO_eps.py b/7.0/models/RL/tensorflow/parallel/PPO_eps.py index 4c11210a..360ce6f6 100644 --- a/7.0/models/RL/tensorflow/parallel/PPO_eps.py +++ b/7.0/models/RL/tensorflow/parallel/PPO_eps.py @@ -28,12 +28,12 @@ def __call__(self,x): class PPO: def __init__(self,state_dim,hidden_dim,action_dim,clip_eps): + self.actor_=actor(state_dim,hidden_dim,action_dim) self.actor=actor(state_dim,hidden_dim,action_dim) - self.nn=actor(state_dim,hidden_dim,action_dim) - nn.assign(self.nn.param,self.actor.param.copy()) + nn.assign(self.actor.param,self.actor_.param) self.critic=critic(state_dim,hidden_dim) self.clip_eps=clip_eps - self.param=[self.actor.param,self.critic.param] + self.param=[self.actor_.param,self.critic.param] self.opt=[o.SGD(param=self.param[0]),o.SGD(param=self.param[1])] self.genv=[gym.make('CartPole-v0') for _ in range(5)] @@ -49,7 +49,7 @@ def env(self,a=None,p=None,initial=None): def loss(self,s,a,next_s,r,d): a=tf.expand_dims(a,axis=1) - raito=tf.gather(self.actor(s),a,axis=1,batch_dims=1)/tf.gather(self.nn(s),a,axis=1,batch_dims=1) + raito=tf.gather(self.actor_(s),a,axis=1,batch_dims=1)/tf.gather(self.actor(s),a,axis=1,batch_dims=1) value=self.critic(s) value_tar=tf.cast(r,'float32')+0.98*self.critic(next_s)*(1-tf.cast(d,'float32')) TD=value_tar-value @@ -60,7 +60,7 @@ def loss(self,s,a,next_s,r,d): def update_param(self): - nn.assign(self.nn.param,self.actor.param) + nn.assign(self.actor.param,self.actor_.param) return From 7e766d1bd3fcc6ee6b430bc3615a86c7684f35ff Mon Sep 17 00:00:00 2001 From: NoteDance Date: Sat, 22 Feb 2025 21:53:15 +0800 Subject: [PATCH 336/337] Add files via upload --- 7.0/models/RL/tensorflow/parallel/DDPG.py | 4 ++-- 7.0/models/RL/tensorflow/parallel/PPO.py | 4 ++-- 7.0/models/RL/tensorflow/parallel/PPO_eps.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/7.0/models/RL/tensorflow/parallel/DDPG.py b/7.0/models/RL/tensorflow/parallel/DDPG.py index 821a9c4a..55b3bbd0 100644 --- a/7.0/models/RL/tensorflow/parallel/DDPG.py +++ b/7.0/models/RL/tensorflow/parallel/DDPG.py @@ -78,6 +78,6 @@ def update_param(self): # update function, kernel uses it to update parameter def opt(self,gradient): # optimization function, kernel uses it to optimize parameter - self.optimizer[0](gradient,self.param[0]) # apply the custom momentum optimizer to update the parameters using the gradient - self.optimizer[1](gradient,self.param[1]) + self.optimizer[0](gradient[0],self.param[0]) # apply the custom momentum optimizer to update the parameters using the gradient + self.optimizer[1](gradient[1],self.param[1]) return self.param \ No newline at end of file diff --git a/7.0/models/RL/tensorflow/parallel/PPO.py b/7.0/models/RL/tensorflow/parallel/PPO.py index dcda703a..751c09f1 100644 --- a/7.0/models/RL/tensorflow/parallel/PPO.py +++ b/7.0/models/RL/tensorflow/parallel/PPO.py @@ -69,6 +69,6 @@ def update_param(self): def opt(self,gradient): - self.optimizer[0](gradient,self.param[0]) - self.optimizer[1](gradient,self.param[1]) + self.optimizer[0](gradient[0],self.param[0]) + self.optimizer[1](gradient[1],self.param[1]) return self.param \ No newline at end of file diff --git a/7.0/models/RL/tensorflow/parallel/PPO_eps.py b/7.0/models/RL/tensorflow/parallel/PPO_eps.py index 360ce6f6..b946e31f 100644 --- a/7.0/models/RL/tensorflow/parallel/PPO_eps.py +++ b/7.0/models/RL/tensorflow/parallel/PPO_eps.py @@ -65,6 +65,6 @@ def update_param(self): def opt(self,gradient): - self.optimizer[0](gradient,self.param[0]) - self.optimizer[1](gradient,self.param[1]) + self.optimizer[0](gradient[0],self.param[0]) + self.optimizer[1](gradient[1],self.param[1]) return self.param \ No newline at end of file From cbb1cce3f8cd5cfa2d3c1f7487e84657e6e269ac Mon Sep 17 00:00:00 2001 From: NoteDance Date: Sat, 22 Feb 2025 22:12:38 +0800 Subject: [PATCH 337/337] Add files via upload --- 7.0/models/RL/tensorflow/parallel/DDPG.py | 6 +++--- 7.0/models/RL/tensorflow/parallel/DQN.py | 4 ++-- 7.0/models/RL/tensorflow/parallel/DQN_pr.py | 4 ++-- 7.0/models/RL/tensorflow/parallel/PPO.py | 6 +++--- 7.0/models/RL/tensorflow/parallel/PPO_eps.py | 6 +++--- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/7.0/models/RL/tensorflow/parallel/DDPG.py b/7.0/models/RL/tensorflow/parallel/DDPG.py index 55b3bbd0..095ae670 100644 --- a/7.0/models/RL/tensorflow/parallel/DDPG.py +++ b/7.0/models/RL/tensorflow/parallel/DDPG.py @@ -78,6 +78,6 @@ def update_param(self): # update function, kernel uses it to update parameter def opt(self,gradient): # optimization function, kernel uses it to optimize parameter - self.optimizer[0](gradient[0],self.param[0]) # apply the custom momentum optimizer to update the parameters using the gradient - self.optimizer[1](gradient[1],self.param[1]) - return self.param \ No newline at end of file + param1=self.optimizer[0](gradient[0],self.param[0]) # apply the custom momentum optimizer to update the parameters using the gradient + param2=self.optimizer[1](gradient[1],self.param[1]) + return [param1,param2] \ No newline at end of file diff --git a/7.0/models/RL/tensorflow/parallel/DQN.py b/7.0/models/RL/tensorflow/parallel/DQN.py index ad7d3738..dcd18861 100644 --- a/7.0/models/RL/tensorflow/parallel/DQN.py +++ b/7.0/models/RL/tensorflow/parallel/DQN.py @@ -47,5 +47,5 @@ def update_param(self): # update function, kernel uses it to update parameter def opt(self,gradient): # optimization function, kernel uses it to optimize parameter - self.optimizer(gradient,self.param) # apply the custom momentum optimizer to update the parameters using the gradient - return self.param \ No newline at end of file + param=self.optimizer(gradient,self.param) # apply the custom momentum optimizer to update the parameters using the gradient + return param \ No newline at end of file diff --git a/7.0/models/RL/tensorflow/parallel/DQN_pr.py b/7.0/models/RL/tensorflow/parallel/DQN_pr.py index b4600a0f..bea2059b 100644 --- a/7.0/models/RL/tensorflow/parallel/DQN_pr.py +++ b/7.0/models/RL/tensorflow/parallel/DQN_pr.py @@ -59,5 +59,5 @@ def update_param(self): # update function, kernel uses it to update parameter def opt(self,gradient): # optimization function, kernel uses it to optimize parameter - self.optimizer(gradient,self.param) # apply the custom momentum optimizer to update the parameters using the gradient - return self.param \ No newline at end of file + param=self.optimizer(gradient,self.param) # apply the custom momentum optimizer to update the parameters using the gradient + return param \ No newline at end of file diff --git a/7.0/models/RL/tensorflow/parallel/PPO.py b/7.0/models/RL/tensorflow/parallel/PPO.py index 751c09f1..8227abe8 100644 --- a/7.0/models/RL/tensorflow/parallel/PPO.py +++ b/7.0/models/RL/tensorflow/parallel/PPO.py @@ -69,6 +69,6 @@ def update_param(self): def opt(self,gradient): - self.optimizer[0](gradient[0],self.param[0]) - self.optimizer[1](gradient[1],self.param[1]) - return self.param \ No newline at end of file + param1=self.optimizer[0](gradient[0],self.param[0]) + param2=self.optimizer[1](gradient[1],self.param[1]) + return [param1,param2] \ No newline at end of file diff --git a/7.0/models/RL/tensorflow/parallel/PPO_eps.py b/7.0/models/RL/tensorflow/parallel/PPO_eps.py index b946e31f..68975a38 100644 --- a/7.0/models/RL/tensorflow/parallel/PPO_eps.py +++ b/7.0/models/RL/tensorflow/parallel/PPO_eps.py @@ -65,6 +65,6 @@ def update_param(self): def opt(self,gradient): - self.optimizer[0](gradient[0],self.param[0]) - self.optimizer[1](gradient[1],self.param[1]) - return self.param \ No newline at end of file + param1=self.optimizer[0](gradient[0],self.param[0]) + param2=self.optimizer[1](gradient[1],self.param[1]) + return [param1,param2] \ No newline at end of file