Internal architecture changes (to allow diferent activation functions for each layer and to allow hidden layers to have different number of nodes).

src/Layer.h

@@ -5,6 +5,7 @@
 #ifndef LAYER_H
 #define LAYER_H
 
+#include "Utils.h"
 #include "Node.h"
 
 #include <stdio.h>
@@ -23,39 +24,59 @@ public:
     m_nodes.clear();
   };
 
-  Layer(int num_nodes,
+  Layer(int num_inputs_per_node,
-        int num_inputs_per_node,
+        int num_nodes,
+        const std::string & activation_function,
         bool use_constant_weight_init = true,
         double constant_weight_init = 0.5) {
-    m_num_nodes = num_nodes;
     m_num_inputs_per_node = num_inputs_per_node;
-    m_nodes.resize(num_nodes);                  
+    m_num_nodes = num_nodes;
+    m_nodes.resize(num_nodes);
 
     for (int i = 0; i < num_nodes; i++) {
       m_nodes[i].WeightInitialization(num_inputs_per_node,
                                       use_constant_weight_init,
                                       constant_weight_init);
     }
+
+    std::pair<std::function<double(double)>,
+      std::function<double(double)> > *pair;
+    bool ret_val = utils::ActivationFunctionsManager::Singleton().
+           GetActivationFunctionPair(activation_function,
+                                     &pair);
+    assert(ret_val);
+    m_activation_function = (*pair).first;
+    m_deriv_activation_function = (*pair).second;
+  };
+
+  ~Layer() {
+    m_num_inputs_per_node = 0;
+    m_num_nodes = 0;
+    m_nodes.clear();
+  };
+
+  int GetInputSize() const {
+    return m_num_inputs_per_node;
   };
   
-  ~Layer() {
+  int GetOutputSize() const {
-    m_num_nodes = 0;
+    return m_num_nodes;
-    m_num_inputs_per_node = 0;
-    m_nodes.clear();
   };
 
   const std::vector<Node> & GetNodes() const {
     return m_nodes;
   }
 
-  void GetOutputAfterSigmoid(const std::vector<double> &input, 
+  void GetOutputAfterActivationFunction(const std::vector<double> &input,
-                             std::vector<double> * output) const {
+                                        std::vector<double> * output) const {
     assert(input.size() == m_num_inputs_per_node);
 
     output->resize(m_num_nodes);
 
     for (int i = 0; i < m_num_nodes; ++i) {
-      m_nodes[i].GetOutputAfterSigmoid(input, &((*output)[i]));
+      m_nodes[i].GetOutputAfterActivationFunction(input,
+                                                  m_activation_function,
+                                                  &((*output)[i]));
     }
   }
 
@@ -79,8 +100,8 @@ public:
       double dnetj_dwij = 0.0;
 
       dE_doj = deriv_error[i];
-      doj_dnetj = utils::deriv_sigmoid(net_sum);
+      doj_dnetj = m_deriv_activation_function(net_sum);
-      
+
       for (int j = 0; j < m_num_inputs_per_node; j++) {
         (*deltas)[j] += dE_doj * doj_dnetj * m_nodes[i].GetWeights()[j];
 
@@ -94,9 +115,12 @@ public:
   };
 
 protected:
-  int m_num_nodes{ 0 };
   int m_num_inputs_per_node{ 0 };
+  int m_num_nodes{ 0 };
   std::vector<Node> m_nodes;
+
+  std::function<double(double)>  m_activation_function;
+  std::function<double(double)>  m_deriv_activation_function;
 };
 
 #endif //LAYER_H

src/MLP.cpp

@@ -27,14 +27,13 @@ void MLP::GetOutput(const std::vector<double> &input,
   if (m_num_hidden_layers == 0)
     temp_size = m_num_outputs;
   else
-    temp_size = m_num_nodes_per_hidden_layer;
+    temp_size = m_layers_nodes[1];
 
   std::vector<double> temp_in(m_num_inputs, 0.0);
   std::vector<double> temp_out(temp_size, 0.0);
   temp_in = input;
 
-  //m_layers.size() equals (m_num_hidden_layers + 1)
+  for (int i = 0; i < m_layers.size(); ++i) {
-  for (int i = 0; i < (m_num_hidden_layers + 1); ++i) {
     if (i > 0) {
       //Store this layer activation
       if (all_layers_activations != nullptr)
@@ -43,11 +42,9 @@ void MLP::GetOutput(const std::vector<double> &input,
       temp_in.clear();
       temp_in = temp_out;
       temp_out.clear();
-      temp_out.resize((i == m_num_hidden_layers) ?
+      temp_out.resize(m_layers[i].GetOutputSize());
-                      m_num_outputs :
-                      m_num_nodes_per_hidden_layer);
     }
-    m_layers[i].GetOutputAfterSigmoid(temp_in, &temp_out);
+    m_layers[i].GetOutputAfterActivationFunction(temp_in, &temp_out);
   }
 
   if (temp_out.size() > 1)
@@ -106,8 +103,9 @@ void MLP::UpdateMiniBatch(const std::vector<TrainingSample> &training_sample_set
   //  }
   //}
   size_t i = 0;
+  double current_iteration_cost_function = 0.0;
   for (i = 0; i < max_iterations; i++) {
-    double current_iteration_cost_function = 0.0;
+    current_iteration_cost_function = 0.0;
     for (auto & training_sample_with_bias : training_sample_set_with_bias) {
       std::vector<double> predicted_output;
       std::vector< std::vector<double> > all_layers_activations;
@@ -153,7 +151,10 @@ void MLP::UpdateMiniBatch(const std::vector<TrainingSample> &training_sample_set
       break;
   }
 
-  LOG(INFO) << "******************************" ;
+  LOG(INFO) << "Iteration " << i << " cost function f(error): "
+    << current_iteration_cost_function;
+
+  LOG(INFO) << "******************************";
   LOG(INFO) << "******* TRAINING ENDED *******";
   LOG(INFO) << "******* " << i << " iters *******";
   LOG(INFO) << "******************************";

src/MLP.h

@@ -19,27 +19,26 @@
 
 class MLP {
 public:
-  MLP(int num_inputs,
+  //desired call sintax :  MLP({64*64,20,4}, {"sigmoid", "linear"},
-      int num_outputs,
+  MLP(const std::vector<uint64_t> & layers_nodes,
-      int num_hidden_layers,
+      const std::vector<std::string> & layers_activfuncs,
-      int num_nodes_per_hidden_layer,
       bool use_constant_weight_init = true,
       double constant_weight_init = 0.5) {
+    assert(layers_nodes.size() >= 2);
+    assert(layers_activfuncs.size() + 1 == layers_nodes.size());
 
-    m_num_inputs = num_inputs;
+    CreateMLP(layers_nodes,
-    m_num_outputs = num_outputs;
+              layers_activfuncs,
-    m_num_hidden_layers = num_hidden_layers;
+              use_constant_weight_init,
-    m_num_nodes_per_hidden_layer = num_nodes_per_hidden_layer;
-
-    CreateMLP(use_constant_weight_init,
               constant_weight_init);
   }
 
+
   ~MLP() {
     m_num_inputs = 0;
     m_num_outputs = 0;
     m_num_hidden_layers = 0;
-    m_num_nodes_per_hidden_layer = 0;
+    m_layers_nodes.clear();
     m_layers.clear();
   };
 
@@ -60,40 +59,27 @@ protected:
                      const std::vector<double> &error,
                      double learning_rate);
 private:
-  void CreateMLP(bool use_constant_weight_init,
+  void CreateMLP(const std::vector<uint64_t> & layers_nodes,
+                 const std::vector<std::string> & layers_activfuncs,
+                 bool use_constant_weight_init,
                  double constant_weight_init = 0.5) {
-    if (m_num_hidden_layers > 0) {
+    m_layers_nodes = layers_nodes;
-      //first layer
+    m_num_inputs = m_layers_nodes[0];
-      m_layers.emplace_back(Layer(m_num_nodes_per_hidden_layer,
+    m_num_outputs = m_layers_nodes[m_layers_nodes.size() - 1];
-                                  m_num_inputs,
+    m_num_hidden_layers = m_layers_nodes.size() - 2;
-                                  use_constant_weight_init,
+
-                                  constant_weight_init));
+    for (int i = 0; i < m_layers_nodes.size() - 1; i++) {
-      //subsequent layers
+      m_layers.emplace_back(Layer(m_layers_nodes[i],
-      for (int i = 0; i < m_num_hidden_layers - 1; i++) {
+                                  m_layers_nodes[i + 1],
-        m_layers.emplace_back(Layer(m_num_nodes_per_hidden_layer,
+                                  layers_activfuncs[i],
-                                    m_num_nodes_per_hidden_layer,
-                                    use_constant_weight_init,
-                                    constant_weight_init));
-      }
-      //last layer
-      m_layers.emplace_back(Layer(m_num_outputs,
-                                  m_num_nodes_per_hidden_layer,
-                                  use_constant_weight_init,
-                                  constant_weight_init));
-    } else {
-      m_layers.emplace_back(Layer(m_num_outputs,
-                                  m_num_inputs,
                                   use_constant_weight_init,
                                   constant_weight_init));
     }
   }
-
-
   int m_num_inputs{ 0 };
   int m_num_outputs{ 0 };
   int m_num_hidden_layers{ 0 };
-  int m_num_nodes_per_hidden_layer{ 0 };
+  std::vector<uint64_t> m_layers_nodes;
-
   std::vector<Layer> m_layers;
 };
 

src/MLPTest.cpp

@@ -39,7 +39,7 @@ UNIT(LearnAND) {
   size_t num_examples = training_sample_set_with_bias.size();
   size_t num_features = training_sample_set_with_bias[0].GetInputVectorSize();
   size_t num_outputs = training_sample_set_with_bias[0].GetOutputVectorSize();
-  MLP my_mlp(num_features, num_outputs, 1, 2, false);
+  MLP my_mlp({ num_features, 2 ,num_outputs }, { "sigmoid", "linear" }, false);
   //Train MLP
   my_mlp.UpdateMiniBatch(training_sample_set_with_bias, 0.5, 500, 0.25);
 
@@ -79,7 +79,7 @@ UNIT(LearnNAND) {
   size_t num_examples = training_sample_set_with_bias.size();
   size_t num_features = training_sample_set_with_bias[0].GetInputVectorSize();
   size_t num_outputs = training_sample_set_with_bias[0].GetOutputVectorSize();
-  MLP my_mlp(num_features, num_outputs, 1, 2, false);
+  MLP my_mlp({ num_features, 2 ,num_outputs }, { "sigmoid", "linear" }, false);
   //Train MLP
   my_mlp.UpdateMiniBatch(training_sample_set_with_bias, 0.5, 500, 0.25);
 
@@ -119,7 +119,7 @@ UNIT(LearnOR) {
   size_t num_examples = training_sample_set_with_bias.size();
   size_t num_features = training_sample_set_with_bias[0].GetInputVectorSize();
   size_t num_outputs = training_sample_set_with_bias[0].GetOutputVectorSize();
-  MLP my_mlp(num_features, num_outputs, 1, 2, false);
+  MLP my_mlp({ num_features, 2 ,num_outputs }, { "sigmoid", "linear" }, false);
   //Train MLP
   my_mlp.UpdateMiniBatch(training_sample_set_with_bias, 0.5, 500, 0.25);
 
@@ -159,7 +159,7 @@ UNIT(LearnNOR) {
   size_t num_examples = training_sample_set_with_bias.size();
   size_t num_features = training_sample_set_with_bias[0].GetInputVectorSize();
   size_t num_outputs = training_sample_set_with_bias[0].GetOutputVectorSize();
-  MLP my_mlp(num_features, num_outputs, 1, 2, false);
+  MLP my_mlp({ num_features, 2 ,num_outputs }, { "sigmoid", "linear" }, false);
   //Train MLP
   my_mlp.UpdateMiniBatch(training_sample_set_with_bias, 0.5, 500, 0.25);
 
@@ -197,9 +197,9 @@ UNIT(LearnXOR) {
   size_t num_examples = training_sample_set_with_bias.size();
   size_t num_features = training_sample_set_with_bias[0].GetInputVectorSize();
   size_t num_outputs = training_sample_set_with_bias[0].GetOutputVectorSize();
-  MLP my_mlp(num_features, num_outputs, 1, 2, false);
+  MLP my_mlp({ num_features, 2 ,num_outputs }, { "sigmoid", "linear" }, false);
   //Train MLP
-  my_mlp.UpdateMiniBatch(training_sample_set_with_bias, 0.5, 50'000, 0.25);
+  my_mlp.UpdateMiniBatch(training_sample_set_with_bias, 0.5, 500, 0.25);
 
   for (const auto & training_sample : training_sample_set_with_bias) {
     std::vector<double>  output;
@@ -233,7 +233,7 @@ UNIT(LearnNOT) {
   size_t num_examples = training_sample_set_with_bias.size();
   size_t num_features = training_sample_set_with_bias[0].GetInputVectorSize();
   size_t num_outputs = training_sample_set_with_bias[0].GetOutputVectorSize();
-  MLP my_mlp(num_features, num_outputs, 1, 2, false);
+  MLP my_mlp({ num_features, 2 ,num_outputs }, { "sigmoid", "linear" }, false);
   //Train MLP
   my_mlp.UpdateMiniBatch(training_sample_set_with_bias, 0.5, 500, 0.25);
 
@@ -271,7 +271,7 @@ UNIT(LearnX1) {
   size_t num_examples = training_sample_set_with_bias.size();
   size_t num_features = training_sample_set_with_bias[0].GetInputVectorSize();
   size_t num_outputs = training_sample_set_with_bias[0].GetOutputVectorSize();
-  MLP my_mlp(num_features, num_outputs, 1, 2, false);
+  MLP my_mlp({ num_features, 2 ,num_outputs }, { "sigmoid", "linear" }, false);
   //Train MLP
   my_mlp.UpdateMiniBatch(training_sample_set_with_bias, 0.5, 500, 0.25);
 
@@ -309,7 +309,7 @@ UNIT(LearnX2) {
   size_t num_examples = training_sample_set_with_bias.size();
   size_t num_features = training_sample_set_with_bias[0].GetInputVectorSize();
   size_t num_outputs = training_sample_set_with_bias[0].GetOutputVectorSize();
-  MLP my_mlp(num_features, num_outputs, 1, 2, false);
+  MLP my_mlp({ num_features, 2 ,num_outputs }, { "sigmoid", "linear" }, false);
   //Train MLP
   my_mlp.UpdateMiniBatch(training_sample_set_with_bias, 0.5, 500, 0.25);
 

src/Node.h

@@ -95,18 +95,20 @@ public:
     *output = inner_prod;
   }
 
-  void GetOutputAfterSigmoid(const std::vector<double> &input,
+  void GetOutputAfterActivationFunction(const std::vector<double> &input,
+                                        std::function<double(double)> activation_function,
                              double * output) const {
     double inner_prod = 0.0;
     GetInputInnerProdWithWeights(input, &inner_prod);
-    *output = utils::sigmoid(inner_prod);
+    *output = activation_function(inner_prod);
   }
 
   void GetBooleanOutput(const std::vector<double> &input,
+                        std::function<double(double)> activation_function,
                         bool * bool_output,
                         double threshold = 0.5) const {
     double value;
-    GetOutputAfterSigmoid(input, &value);
+    GetOutputAfterActivationFunction(input, activation_function, &value);
     *bool_output = (value >threshold) ? true : false;
   };
 

src/NodeTest.cpp

@@ -39,7 +39,10 @@ void Train(Node & node,
     int error_count = 0;
     for (auto & training_sample_with_bias : training_sample_set_with_bias) {
       bool prediction;
-      node.GetBooleanOutput(training_sample_with_bias.input_vector(), &prediction, 0.5);
+      node.GetBooleanOutput(training_sample_with_bias.input_vector(),
+                            utils::linear,
+                            &prediction, 
+                            0.5);
       bool correct_output = training_sample_with_bias.output_vector()[0] > 0.5 ? true : false;
       if (prediction != correct_output) {
         error_count++;
@@ -85,7 +88,10 @@ UNIT(LearnAND) {
 
   for (const auto & training_sample : training_sample_set_with_bias) {
     bool class_id;
-    my_node.GetBooleanOutput(training_sample.input_vector(), &class_id, 0.5);
+    my_node.GetBooleanOutput(training_sample.input_vector(),
+                             utils::linear,
+                             &class_id, 
+                             0.5);
     bool correct_output = training_sample.output_vector()[0] > 0.5 ? true : false;
     ASSERT_TRUE(class_id == correct_output);
   }
@@ -117,7 +123,10 @@ UNIT(LearnNAND) {
 
   for (const auto & training_sample : training_sample_set_with_bias) {
     bool class_id;
-    my_node.GetBooleanOutput(training_sample.input_vector(), &class_id, 0.5);
+    my_node.GetBooleanOutput(training_sample.input_vector(),
+                             utils::linear,
+                             &class_id,
+                             0.5);
     bool correct_output = training_sample.output_vector()[0] > 0.5 ? true : false;
     ASSERT_TRUE(class_id == correct_output);
   }
@@ -149,7 +158,10 @@ UNIT(LearnOR) {
 
   for (const auto & training_sample : training_sample_set_with_bias) {
     bool class_id;
-    my_node.GetBooleanOutput(training_sample.input_vector(), &class_id, 0.5);
+    my_node.GetBooleanOutput(training_sample.input_vector(),
+                             utils::linear,
+                             &class_id,
+                             0.5);
     bool correct_output = training_sample.output_vector()[0] > 0.5 ? true : false;
     ASSERT_TRUE(class_id == correct_output);
   }
@@ -180,7 +192,10 @@ UNIT(LearnNOR) {
 
   for (const auto & training_sample : training_sample_set_with_bias) {
     bool class_id;
-    my_node.GetBooleanOutput(training_sample.input_vector(), &class_id, 0.5);
+    my_node.GetBooleanOutput(training_sample.input_vector(),
+                             utils::linear,
+                             &class_id,
+                             0.5);
     bool correct_output = training_sample.output_vector()[0] > 0.5 ? true : false;
     ASSERT_TRUE(class_id == correct_output);
   }
@@ -210,7 +225,10 @@ UNIT(LearnNOT) {
 
   for (const auto & training_sample : training_sample_set_with_bias) {
     bool class_id;
-    my_node.GetBooleanOutput(training_sample.input_vector(), &class_id, 0.5);
+    my_node.GetBooleanOutput(training_sample.input_vector(),
+                             utils::linear,
+                             &class_id,
+                             0.5);
     bool correct_output = training_sample.output_vector()[0] > 0.5 ? true : false;
     ASSERT_TRUE(class_id == correct_output);
   }
@@ -242,7 +260,10 @@ UNIT(LearnXOR) {
 
   for (const auto & training_sample : training_sample_set_with_bias) {
     bool class_id;
-    my_node.GetBooleanOutput(training_sample.input_vector(), &class_id, 0.5);
+    my_node.GetBooleanOutput(training_sample.input_vector(),
+                             utils::linear,
+                             &class_id,
+                             0.5);
     bool correct_output = training_sample.output_vector()[0] > 0.5 ? true : false;
     if (class_id != correct_output) {
       LOG(WARNING) << "Failed to train. " <<

src/Utils.h

@@ -10,20 +10,91 @@
 #include <math.h>
 #include <numeric>
 #include <chrono>
+#include <string>
+#include <algorithm>
+#include <iostream>
+#include <sstream>
+#include <iterator>
+#include <unordered_map>
+#include <vector>
+#include <cmath>
+#include <functional>
+#include <typeinfo>
+#include <typeindex>
+#include <cassert>
 #ifdef _WIN32
 #include <time.h>
 #else
 #include <sys/time.h>
 #endif
 
-#include <algorithm>
-#include <iostream>
-#include <sstream>
-#include <iterator>
-#include <vector>
-#include <cmath>
-
 namespace utils {
+//Typical sigmoid function created from input x
+//Returns the sigmoided value
+inline double sigmoid(double x) {
+  return 1 / (1 + exp(-x));
+}
+
+// Derivative of sigmoid function
+inline double deriv_sigmoid(double x) {
+  return sigmoid(x)*(1 - sigmoid(x));
+};
+
+//Compute hyperbolic tangent (tanh)
+//Returns the hyperbolic tangent of x.
+inline double hyperbolic_tan(double x) {
+  return (tanh)(x);
+}
+
+// Derivative of hyperbolic tangent function
+inline double deriv_hyperbolic_tan(double x) {
+  return 1 - (std::pow)(hyperbolic_tan(x), 2);
+};
+
+inline double linear(double x) {
+  return x;
+}
+
+// Derivative of linear function
+inline double deriv_linear(double x) {
+  return 1;
+};
+
+struct ActivationFunctionsManager {
+  bool GetActivationFunctionPair(const std::string & activation_name,
+                                    std::pair<std::function<double(double)>,
+                                    std::function<double(double)> > **pair) {
+    auto iter = activation_functions_map.find(activation_name);
+    if (iter != activation_functions_map.end())
+      *pair = &(iter->second);
+    else
+      return false;
+    return true;
+  }
+
+  static ActivationFunctionsManager & Singleton() {
+    static ActivationFunctionsManager instance;
+    return instance;
+  }
+private:
+  void AddNewPair(std::string function_name,
+                  std::function<double(double)> function,
+                  std::function<double(double)> deriv_function) {
+    activation_functions_map.insert(std::make_pair(function_name,
+                                                   std::make_pair(function,
+                                                                  deriv_function)));
+  };
+
+  ActivationFunctionsManager() {
+    AddNewPair("sigmoid", sigmoid, deriv_sigmoid);
+    AddNewPair("tanh", hyperbolic_tan, deriv_hyperbolic_tan);
+    AddNewPair("linear", linear, deriv_linear);
+  };
+
+  std::unordered_map<std::string,
+    std::pair<std::function<double(double)>, std::function<double(double)> > >
+    activation_functions_map;
+};
 
 struct gen_rand {
   double factor;
@@ -35,18 +106,6 @@ public:
   }
 };
 
-inline double sigmoid(double x) {
-  //Typical sigmoid function created from input x
-  //param x: input value
-  //return: sigmoided value
-  return 1 / (1 + exp(-x));
-}
-
-// Derivative of sigmoid function
-inline double deriv_sigmoid(double x) {
-  return sigmoid(x)*(1 - sigmoid(x));
-};
-
 inline void Softmax(std::vector<double> *output) {
   size_t num_elements = output->size();
   std::vector<double> exp_output(num_elements);
@@ -62,8 +121,8 @@ inline void Softmax(std::vector<double> *output) {
 
 inline void  GetIdMaxElement(const std::vector<double> &output, size_t * class_id) {
   *class_id = std::distance(output.begin(),
-                                   std::max_element(output.begin(),
+                            std::max_element(output.begin(),
-                                                    output.end()));
+                                             output.end()));
 }
 }
 #endif // UTILS_H