Mirrors/MLP
1
0
Fork 0
mirror of https://github.com/davidalbertonogueira/MLP.git synced 2025-12-17 04:14:41 +03:00
Code Issues Actions Packages Projects Releases Wiki Activity

Internal architecture changes (to allow diferent activation functions for each layer and to allow hidden layers to have different number of nodes).

Browse Source
This commit is contained in:
davidjacnogueira
2016-11-09 02:31:53 +00:00
parent f647b05f70
commit 0a636416ed
7 changed files with 193 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
src/Layer.h
View File

@@ -5,6 +5,7 @@
#ifndef LAYER_H
#define LAYER_H
#include "Utils.h"
#include "Node.h"
#include <stdio.h>
@@ -23,12 +24,13 @@ public:
m_nodes.clear();
};
Layer(int num_nodes,
int num_inputs_per_node,
Layer(int num_inputs_per_node,
int num_nodes,
const std::string & activation_function,
bool use_constant_weight_init = true,
double constant_weight_init = 0.5) {
m_num_nodes = num_nodes;
m_num_inputs_per_node = num_inputs_per_node;
m_num_nodes = num_nodes;
m_nodes.resize(num_nodes);
for (int i = 0; i < num_nodes; i++) {
@@ -36,26 +38,45 @@ public:
use_constant_weight_init,
constant_weight_init);
}
std::pair<std::function<double(double)>,
std::function<double(double)> > *pair;
bool ret_val = utils::ActivationFunctionsManager::Singleton().
GetActivationFunctionPair(activation_function,
&pair);
assert(ret_val);
m_activation_function = (*pair).first;
m_deriv_activation_function = (*pair).second;
};
~Layer() {
m_num_nodes = 0;
m_num_inputs_per_node = 0;
m_num_nodes = 0;
m_nodes.clear();
};
int GetInputSize() const {
return m_num_inputs_per_node;
};
int GetOutputSize() const {
return m_num_nodes;
};
const std::vector<Node> & GetNodes() const {
return m_nodes;
}
void GetOutputAfterSigmoid(const std::vector<double> &input,
void GetOutputAfterActivationFunction(const std::vector<double> &input,
std::vector<double> * output) const {
assert(input.size() == m_num_inputs_per_node);
output->resize(m_num_nodes);
for (int i = 0; i < m_num_nodes; ++i) {
m_nodes[i].GetOutputAfterSigmoid(input, &((*output)[i]));
m_nodes[i].GetOutputAfterActivationFunction(input,
m_activation_function,
&((*output)[i]));
}
}
@@ -79,7 +100,7 @@ public:
double dnetj_dwij = 0.0;
dE_doj = deriv_error[i];
doj_dnetj = utils::deriv_sigmoid(net_sum);
doj_dnetj = m_deriv_activation_function(net_sum);
for (int j = 0; j < m_num_inputs_per_node; j++) {
(*deltas)[j] += dE_doj * doj_dnetj * m_nodes[i].GetWeights()[j];
@@ -94,9 +115,12 @@ public:
};
protected:
int m_num_nodes{ 0 };
int m_num_inputs_per_node{ 0 };
int m_num_nodes{ 0 };
std::vector<Node> m_nodes;
std::function<double(double)> m_activation_function;
std::function<double(double)> m_deriv_activation_function;
};
#endif //LAYER_H

17
src/MLP.cpp
View File

@@ -27,14 +27,13 @@ void MLP::GetOutput(const std::vector<double> &input,
if (m_num_hidden_layers == 0)
temp_size = m_num_outputs;
else
temp_size = m_num_nodes_per_hidden_layer;
temp_size = m_layers_nodes[1];
std::vector<double> temp_in(m_num_inputs, 0.0);
std::vector<double> temp_out(temp_size, 0.0);
temp_in = input;
//m_layers.size() equals (m_num_hidden_layers + 1)
for (int i = 0; i < (m_num_hidden_layers + 1); ++i) {
for (int i = 0; i < m_layers.size(); ++i) {
if (i > 0) {
//Store this layer activation
if (all_layers_activations != nullptr)
@@ -43,11 +42,9 @@ void MLP::GetOutput(const std::vector<double> &input,
temp_in.clear();
temp_in = temp_out;
temp_out.clear();
temp_out.resize((i == m_num_hidden_layers) ?
m_num_outputs :
m_num_nodes_per_hidden_layer);
temp_out.resize(m_layers[i].GetOutputSize());
}
m_layers[i].GetOutputAfterSigmoid(temp_in, &temp_out);
m_layers[i].GetOutputAfterActivationFunction(temp_in, &temp_out);
}
if (temp_out.size() > 1)
@@ -106,8 +103,9 @@ void MLP::UpdateMiniBatch(const std::vector<TrainingSample> &training_sample_set
// }
//}
size_t i = 0;
for (i = 0; i < max_iterations; i++) {
double current_iteration_cost_function = 0.0;
for (i = 0; i < max_iterations; i++) {
current_iteration_cost_function = 0.0;
for (auto & training_sample_with_bias : training_sample_set_with_bias) {
std::vector<double> predicted_output;
std::vector< std::vector<double> > all_layers_activations;
@@ -153,6 +151,9 @@ void MLP::UpdateMiniBatch(const std::vector<TrainingSample> &training_sample_set
break;
}
LOG(INFO) << "Iteration " << i << " cost function f(error): "
<< current_iteration_cost_function;
LOG(INFO) << "******************************";
LOG(INFO) << "******* TRAINING ENDED *******";
LOG(INFO) << "******* " << i << " iters *******";

66
src/MLP.h
View File

@@ -19,27 +19,26 @@
class MLP {
public:
MLP(int num_inputs,
int num_outputs,
int num_hidden_layers,
int num_nodes_per_hidden_layer,
//desired call sintax : MLP({64*64,20,4}, {"sigmoid", "linear"},
MLP(const std::vector<uint64_t> & layers_nodes,
const std::vector<std::string> & layers_activfuncs,
bool use_constant_weight_init = true,
double constant_weight_init = 0.5) {
assert(layers_nodes.size() >= 2);
assert(layers_activfuncs.size() + 1 == layers_nodes.size());
m_num_inputs = num_inputs;
m_num_outputs = num_outputs;
m_num_hidden_layers = num_hidden_layers;
m_num_nodes_per_hidden_layer = num_nodes_per_hidden_layer;
CreateMLP(use_constant_weight_init,
CreateMLP(layers_nodes,
layers_activfuncs,
use_constant_weight_init,
constant_weight_init);
}
~MLP() {
m_num_inputs = 0;
m_num_outputs = 0;
m_num_hidden_layers = 0;
m_num_nodes_per_hidden_layer = 0;
m_layers_nodes.clear();
m_layers.clear();
};
@@ -60,40 +59,27 @@ protected:
const std::vector<double> &error,
double learning_rate);
private:
void CreateMLP(bool use_constant_weight_init,
void CreateMLP(const std::vector<uint64_t> & layers_nodes,
const std::vector<std::string> & layers_activfuncs,
bool use_constant_weight_init,
double constant_weight_init = 0.5) {
if (m_num_hidden_layers > 0) {
//first layer
m_layers.emplace_back(Layer(m_num_nodes_per_hidden_layer,
m_num_inputs,
use_constant_weight_init,
constant_weight_init));
//subsequent layers
for (int i = 0; i < m_num_hidden_layers - 1; i++) {
m_layers.emplace_back(Layer(m_num_nodes_per_hidden_layer,
m_num_nodes_per_hidden_layer,
use_constant_weight_init,
constant_weight_init));
}
//last layer
m_layers.emplace_back(Layer(m_num_outputs,
m_num_nodes_per_hidden_layer,
use_constant_weight_init,
constant_weight_init));
} else {
m_layers.emplace_back(Layer(m_num_outputs,
m_num_inputs,
use_constant_weight_init,
constant_weight_init));
}
}
m_layers_nodes = layers_nodes;
m_num_inputs = m_layers_nodes[0];
m_num_outputs = m_layers_nodes[m_layers_nodes.size() - 1];
m_num_hidden_layers = m_layers_nodes.size() - 2;
for (int i = 0; i < m_layers_nodes.size() - 1; i++) {
m_layers.emplace_back(Layer(m_layers_nodes[i],
m_layers_nodes[i + 1],
layers_activfuncs[i],
use_constant_weight_init,
constant_weight_init));
}
}
int m_num_inputs{ 0 };
int m_num_outputs{ 0 };
int m_num_hidden_layers{ 0 };
int m_num_nodes_per_hidden_layer{ 0 };
std::vector<uint64_t> m_layers_nodes;
std::vector<Layer> m_layers;
};

18
src/MLPTest.cpp
View File

@@ -39,7 +39,7 @@ UNIT(LearnAND) {
size_t num_examples = training_sample_set_with_bias.size();
size_t num_features = training_sample_set_with_bias[0].GetInputVectorSize();
size_t num_outputs = training_sample_set_with_bias[0].GetOutputVectorSize();
MLP my_mlp(num_features, num_outputs, 1, 2, false);
MLP my_mlp({ num_features, 2 ,num_outputs }, { "sigmoid", "linear" }, false);
//Train MLP
my_mlp.UpdateMiniBatch(training_sample_set_with_bias, 0.5, 500, 0.25);
@@ -79,7 +79,7 @@ UNIT(LearnNAND) {
size_t num_examples = training_sample_set_with_bias.size();
size_t num_features = training_sample_set_with_bias[0].GetInputVectorSize();
size_t num_outputs = training_sample_set_with_bias[0].GetOutputVectorSize();
MLP my_mlp(num_features, num_outputs, 1, 2, false);
MLP my_mlp({ num_features, 2 ,num_outputs }, { "sigmoid", "linear" }, false);
//Train MLP
my_mlp.UpdateMiniBatch(training_sample_set_with_bias, 0.5, 500, 0.25);
@@ -119,7 +119,7 @@ UNIT(LearnOR) {
size_t num_examples = training_sample_set_with_bias.size();
size_t num_features = training_sample_set_with_bias[0].GetInputVectorSize();
size_t num_outputs = training_sample_set_with_bias[0].GetOutputVectorSize();
MLP my_mlp(num_features, num_outputs, 1, 2, false);
MLP my_mlp({ num_features, 2 ,num_outputs }, { "sigmoid", "linear" }, false);
//Train MLP
my_mlp.UpdateMiniBatch(training_sample_set_with_bias, 0.5, 500, 0.25);
@@ -159,7 +159,7 @@ UNIT(LearnNOR) {
size_t num_examples = training_sample_set_with_bias.size();
size_t num_features = training_sample_set_with_bias[0].GetInputVectorSize();
size_t num_outputs = training_sample_set_with_bias[0].GetOutputVectorSize();
MLP my_mlp(num_features, num_outputs, 1, 2, false);
MLP my_mlp({ num_features, 2 ,num_outputs }, { "sigmoid", "linear" }, false);
//Train MLP
my_mlp.UpdateMiniBatch(training_sample_set_with_bias, 0.5, 500, 0.25);
@@ -197,9 +197,9 @@ UNIT(LearnXOR) {
size_t num_examples = training_sample_set_with_bias.size();
size_t num_features = training_sample_set_with_bias[0].GetInputVectorSize();
size_t num_outputs = training_sample_set_with_bias[0].GetOutputVectorSize();
MLP my_mlp(num_features, num_outputs, 1, 2, false);
MLP my_mlp({ num_features, 2 ,num_outputs }, { "sigmoid", "linear" }, false);
//Train MLP
my_mlp.UpdateMiniBatch(training_sample_set_with_bias, 0.5, 50'000, 0.25);
my_mlp.UpdateMiniBatch(training_sample_set_with_bias, 0.5, 500, 0.25);
for (const auto & training_sample : training_sample_set_with_bias) {
std::vector<double> output;
@@ -233,7 +233,7 @@ UNIT(LearnNOT) {
size_t num_examples = training_sample_set_with_bias.size();
size_t num_features = training_sample_set_with_bias[0].GetInputVectorSize();
size_t num_outputs = training_sample_set_with_bias[0].GetOutputVectorSize();
MLP my_mlp(num_features, num_outputs, 1, 2, false);
MLP my_mlp({ num_features, 2 ,num_outputs }, { "sigmoid", "linear" }, false);
//Train MLP
my_mlp.UpdateMiniBatch(training_sample_set_with_bias, 0.5, 500, 0.25);
@@ -271,7 +271,7 @@ UNIT(LearnX1) {
size_t num_examples = training_sample_set_with_bias.size();
size_t num_features = training_sample_set_with_bias[0].GetInputVectorSize();
size_t num_outputs = training_sample_set_with_bias[0].GetOutputVectorSize();
MLP my_mlp(num_features, num_outputs, 1, 2, false);
MLP my_mlp({ num_features, 2 ,num_outputs }, { "sigmoid", "linear" }, false);
//Train MLP
my_mlp.UpdateMiniBatch(training_sample_set_with_bias, 0.5, 500, 0.25);
@@ -309,7 +309,7 @@ UNIT(LearnX2) {
size_t num_examples = training_sample_set_with_bias.size();
size_t num_features = training_sample_set_with_bias[0].GetInputVectorSize();
size_t num_outputs = training_sample_set_with_bias[0].GetOutputVectorSize();
MLP my_mlp(num_features, num_outputs, 1, 2, false);
MLP my_mlp({ num_features, 2 ,num_outputs }, { "sigmoid", "linear" }, false);
//Train MLP
my_mlp.UpdateMiniBatch(training_sample_set_with_bias, 0.5, 500, 0.25);

8
src/Node.h
View File

@@ -95,18 +95,20 @@ public:
*output = inner_prod;
}
void GetOutputAfterSigmoid(const std::vector<double> &input,
void GetOutputAfterActivationFunction(const std::vector<double> &input,
std::function<double(double)> activation_function,
double * output) const {
double inner_prod = 0.0;
GetInputInnerProdWithWeights(input, &inner_prod);
*output = utils::sigmoid(inner_prod);
*output = activation_function(inner_prod);
}
void GetBooleanOutput(const std::vector<double> &input,
std::function<double(double)> activation_function,
bool * bool_output,
double threshold = 0.5) const {
double value;
GetOutputAfterSigmoid(input, &value);
GetOutputAfterActivationFunction(input, activation_function, &value);
*bool_output = (value >threshold) ? true : false;
};

35
src/NodeTest.cpp
View File

@@ -39,7 +39,10 @@ void Train(Node & node,
int error_count = 0;
for (auto & training_sample_with_bias : training_sample_set_with_bias) {
bool prediction;
node.GetBooleanOutput(training_sample_with_bias.input_vector(), &prediction, 0.5);
node.GetBooleanOutput(training_sample_with_bias.input_vector(),
utils::linear,
&prediction,
0.5);
bool correct_output = training_sample_with_bias.output_vector()[0] > 0.5 ? true : false;
if (prediction != correct_output) {
error_count++;
@@ -85,7 +88,10 @@ UNIT(LearnAND) {
for (const auto & training_sample : training_sample_set_with_bias) {
bool class_id;
my_node.GetBooleanOutput(training_sample.input_vector(), &class_id, 0.5);
my_node.GetBooleanOutput(training_sample.input_vector(),
utils::linear,
&class_id,
0.5);
bool correct_output = training_sample.output_vector()[0] > 0.5 ? true : false;
ASSERT_TRUE(class_id == correct_output);
}
@@ -117,7 +123,10 @@ UNIT(LearnNAND) {
for (const auto & training_sample : training_sample_set_with_bias) {
bool class_id;
my_node.GetBooleanOutput(training_sample.input_vector(), &class_id, 0.5);
my_node.GetBooleanOutput(training_sample.input_vector(),
utils::linear,
&class_id,
0.5);
bool correct_output = training_sample.output_vector()[0] > 0.5 ? true : false;
ASSERT_TRUE(class_id == correct_output);
}
@@ -149,7 +158,10 @@ UNIT(LearnOR) {
for (const auto & training_sample : training_sample_set_with_bias) {
bool class_id;
my_node.GetBooleanOutput(training_sample.input_vector(), &class_id, 0.5);
my_node.GetBooleanOutput(training_sample.input_vector(),
utils::linear,
&class_id,
0.5);
bool correct_output = training_sample.output_vector()[0] > 0.5 ? true : false;
ASSERT_TRUE(class_id == correct_output);
}
@@ -180,7 +192,10 @@ UNIT(LearnNOR) {
for (const auto & training_sample : training_sample_set_with_bias) {
bool class_id;
my_node.GetBooleanOutput(training_sample.input_vector(), &class_id, 0.5);
my_node.GetBooleanOutput(training_sample.input_vector(),
utils::linear,
&class_id,
0.5);
bool correct_output = training_sample.output_vector()[0] > 0.5 ? true : false;
ASSERT_TRUE(class_id == correct_output);
}
@@ -210,7 +225,10 @@ UNIT(LearnNOT) {
for (const auto & training_sample : training_sample_set_with_bias) {
bool class_id;
my_node.GetBooleanOutput(training_sample.input_vector(), &class_id, 0.5);
my_node.GetBooleanOutput(training_sample.input_vector(),
utils::linear,
&class_id,
0.5);
bool correct_output = training_sample.output_vector()[0] > 0.5 ? true : false;
ASSERT_TRUE(class_id == correct_output);
}
@@ -242,7 +260,10 @@ UNIT(LearnXOR) {
for (const auto & training_sample : training_sample_set_with_bias) {
bool class_id;
my_node.GetBooleanOutput(training_sample.input_vector(), &class_id, 0.5);
my_node.GetBooleanOutput(training_sample.input_vector(),
utils::linear,
&class_id,
0.5);
bool correct_output = training_sample.output_vector()[0] > 0.5 ? true : false;
if (class_id != correct_output) {
LOG(WARNING) << "Failed to train. " <<

97
src/Utils.h
View File

@@ -10,20 +10,91 @@
#include <math.h>
#include <numeric>
#include <chrono>
#include <string>
#include <algorithm>
#include <iostream>
#include <sstream>
#include <iterator>
#include <unordered_map>
#include <vector>
#include <cmath>
#include <functional>
#include <typeinfo>
#include <typeindex>
#include <cassert>
#ifdef _WIN32
#include <time.h>
#else
#include <sys/time.h>
#endif
#include <algorithm>
#include <iostream>
#include <sstream>
#include <iterator>
#include <vector>
#include <cmath>
namespace utils {
//Typical sigmoid function created from input x
//Returns the sigmoided value
inline double sigmoid(double x) {
return 1 / (1 + exp(-x));
}
// Derivative of sigmoid function
inline double deriv_sigmoid(double x) {
return sigmoid(x)*(1 - sigmoid(x));
};
//Compute hyperbolic tangent (tanh)
//Returns the hyperbolic tangent of x.
inline double hyperbolic_tan(double x) {
return (tanh)(x);
}
// Derivative of hyperbolic tangent function
inline double deriv_hyperbolic_tan(double x) {
return 1 - (std::pow)(hyperbolic_tan(x), 2);
};
inline double linear(double x) {
return x;
}
// Derivative of linear function
inline double deriv_linear(double x) {
return 1;
};
struct ActivationFunctionsManager {
bool GetActivationFunctionPair(const std::string & activation_name,
std::pair<std::function<double(double)>,
std::function<double(double)> > **pair) {
auto iter = activation_functions_map.find(activation_name);
if (iter != activation_functions_map.end())
*pair = &(iter->second);
else
return false;
return true;
}
static ActivationFunctionsManager & Singleton() {
static ActivationFunctionsManager instance;
return instance;
}
private:
void AddNewPair(std::string function_name,
std::function<double(double)> function,
std::function<double(double)> deriv_function) {
activation_functions_map.insert(std::make_pair(function_name,
std::make_pair(function,
deriv_function)));
};
ActivationFunctionsManager() {
AddNewPair("sigmoid", sigmoid, deriv_sigmoid);
AddNewPair("tanh", hyperbolic_tan, deriv_hyperbolic_tan);
AddNewPair("linear", linear, deriv_linear);
};
std::unordered_map<std::string,
std::pair<std::function<double(double)>, std::function<double(double)> > >
activation_functions_map;
};
struct gen_rand {
double factor;
@@ -35,18 +106,6 @@ public:
}
};
inline double sigmoid(double x) {
//Typical sigmoid function created from input x
//param x: input value
//return: sigmoided value
return 1 / (1 + exp(-x));
}
// Derivative of sigmoid function
inline double deriv_sigmoid(double x) {
return sigmoid(x)*(1 - sigmoid(x));
};
inline void Softmax(std::vector<double> *output) {
size_t num_elements = output->size();
std::vector<double> exp_output(num_elements);