NeuralNetwork Class Reference

#include <NeuralNetwork.h>

Inheritance diagram for NeuralNetwork:

List of all members.

Public Member Functions
	NeuralNetwork ()
	~NeuralNetwork ()
void	demo ()
virtual void	modelInit ()
virtual void	modelUpdate (REAL *input, REAL *target, uint nSamples, uint crossRun)
virtual void	predictAllOutputs (REAL *rawInputs, REAL *outputs, uint nSamples, uint crossRun)
virtual void	readSpecificMaps ()
virtual void	saveWeights (int cross)
virtual void	loadWeights (int cross)
virtual void	loadMetaWeights (int cross)
Static Public Member Functions
static string	templateGenerator (int id, string preEffect, int nameID, bool blendStop)
Private Attributes
REAL **	m_inputs
NN **	m_nn
int	m_epoch
int	m_nrLayer
int	m_batchSize
double	m_offsetOutputs
double	m_scaleOutputs
double	m_initWeightFactor
double	m_learnrate
double	m_learnrateMinimum
double	m_learnrateSubtractionValueAfterEverySample
double	m_learnrateSubtractionValueAfterEveryEpoch
double	m_momentum
double	m_weightDecay
double	m_minUpdateErrorBound
double	m_etaPosRPROP
double	m_etaNegRPROP
double	m_minUpdateRPROP
double	m_maxUpdateRPROP
bool	m_enableL1Regularization
bool	m_enableErrorFunctionMAE
bool	m_enableRPROP
bool	m_useBLASforTraining
string	m_neuronsPerLayer
string	m_activationFunction

---

Detailed Description

Basic NeuralNetwork for real-valued prediction The Algorithm makes use of the NN class. In the NN class, there is the complete NN defined, this class is just for data preparation around

Definition at line 17 of file NeuralNetwork.h.

---

Constructor & Destructor Documentation

NeuralNetwork::NeuralNetwork

(

)

Constructor

Definition at line 8 of file NeuralNetwork.cpp.

{
    cout<<"NeuralNetwork"<<endl;
    // init member vars
    m_inputs = 0;
    m_nn = 0;
    m_epoch = 0;
    m_nrLayer = 0;
    m_batchSize = 0;
    m_offsetOutputs = 0;
    m_scaleOutputs = 0;
    m_initWeightFactor = 0;
    m_learnrate = 0;
    m_learnrateMinimum = 0;
    m_learnrateSubtractionValueAfterEverySample = 0;
    m_learnrateSubtractionValueAfterEveryEpoch = 0;
    m_momentum = 0;
    m_weightDecay = 0;
    m_minUpdateErrorBound = 0;
    m_etaPosRPROP = 0;
    m_etaNegRPROP = 0;
    m_minUpdateRPROP = 0;
    m_maxUpdateRPROP = 0;
    m_enableRPROP = 0;
    m_useBLASforTraining = 0;
    m_enableL1Regularization = 0;
    m_enableErrorFunctionMAE = 0;
}

NeuralNetwork::~NeuralNetwork

(

)

Destructor

Definition at line 40 of file NeuralNetwork.cpp.

{
    cout<<"descructor NeuralNetwork"<<endl;

    for ( int i=0;i<m_nCross+1;i++ )
    {
        if ( m_nn[i] )
            delete m_nn[i];
        m_nn[i] = 0;
    }
    delete[] m_nn;

}

---

Member Function Documentation

void NeuralNetwork::demo

(

)

Demonstrate the ability to learn a random function from a Neural Network Is a good example for NN setup and training

Definition at line 92 of file NeuralNetwork.cpp.

{
    int inputs = 784, targets = 10, exampleTrain = 60000, exampleTest = 100;
    REAL* trainInput = new REAL[inputs * exampleTrain];
    REAL* testInput = new REAL[inputs * exampleTest];
    REAL* trainTarget = new REAL[targets * exampleTrain];
    REAL* testTarget = new REAL[targets * exampleTest];

    for ( int i=0;i<inputs*exampleTrain;i++ )
        trainInput[i] = ( double ) rand() / ( double ) RAND_MAX - 0.5;
    for ( int i=0;i<inputs*exampleTest;i++ )
        testInput[i] = ( double ) rand() / ( double ) RAND_MAX - 0.5;
    for ( int i=0;i<targets*exampleTrain;i++ )
        trainTarget[i] = ( double ) rand() / ( double ) RAND_MAX - 0.5;
    for ( int i=0;i<targets*exampleTest;i++ )
        testTarget[i] = ( double ) rand() / ( double ) RAND_MAX - 0.5;

    NN nn;
    nn.setNrTargets ( targets );
    nn.setNrInputs ( inputs );
    nn.setNrExamplesTrain ( exampleTrain );
    nn.setNrExamplesProbe ( exampleTest );
    nn.setTrainInputs ( trainInput );
    nn.setTrainTargets ( trainTarget );
    nn.setProbeInputs ( testInput );
    nn.setProbeTargets ( testTarget );

    // learn parameters
    nn.setInitWeightFactor ( m_initWeightFactor );
    nn.setLearnrate ( m_learnrate );
    nn.setLearnrateMinimum ( m_learnrateMinimum );
    nn.setLearnrateSubtractionValueAfterEverySample ( m_learnrateSubtractionValueAfterEverySample );
    nn.setLearnrateSubtractionValueAfterEveryEpoch ( m_learnrateSubtractionValueAfterEveryEpoch );
    nn.setMomentum ( m_momentum );
    nn.setWeightDecay ( m_weightDecay );
    nn.setMinUpdateErrorBound ( m_minUpdateErrorBound );
    nn.setBatchSize ( m_batchSize );
    nn.setMaxEpochs ( m_maxTuninigEpochs );
    nn.setL1Regularization ( m_enableL1Regularization );
    nn.enableErrorFunctionMAE ( m_enableErrorFunctionMAE );

    // set net inner stucture
    int nrLayer = m_nrLayer;
    int* neuronsPerLayer = Data::splitStringToIntegerList ( m_neuronsPerLayer, ',' );
    nn.enableRPROP ( m_enableRPROP );
    nn.setNNStructure ( nrLayer, neuronsPerLayer );
    nn.useBLASforTraining ( m_useBLASforTraining );
    nn.setScaleOffset ( 1.0, 0.0 );
    nn.setRPROPPosNeg ( m_etaPosRPROP, m_etaNegRPROP );
    nn.setRPROPMinMaxUpdate ( m_minUpdateRPROP, m_maxUpdateRPROP );
    nn.setNormalTrainStopping ( true );
    nn.initNNWeights ( 0 );
    nn.setActivationFunctionType( 0 );
    delete[] neuronsPerLayer;

    // training
    nn.trainNN();

    delete[] trainInput;
    delete[] testInput;
    delete[] trainTarget;
    delete[] testTarget;
}

void NeuralNetwork::loadMetaWeights

(

int

cross

)

[virtual]

nothing to do in a gradient-descent based algorithm

Implements StandardAlgorithm.

Definition at line 405 of file NeuralNetwork.cpp.

{
    // nothing to do in a gradient-descent based algorithm
}

void NeuralNetwork::loadWeights

(

int

cross

)

[virtual]

Load the weights and all other parameters and make the model ready to predict

Implements StandardAlgorithm.

Definition at line 320 of file NeuralNetwork.cpp.

{
    // load weights
    char buf[1024];
    sprintf ( buf,"%02d",cross );
    string name = m_datasetPath + "/" + m_tempPath + "/" + m_weightFile + "." + buf;
    cout<<"Load:"<<name<<endl;
    fstream f ( name.c_str(), ios::in );
    if ( f.is_open() == false )
        assert ( false );
    f.read ( ( char* ) &m_nTrain, sizeof ( int ) );
    f.read ( ( char* ) &m_nFeatures, sizeof ( int ) );
    f.read ( ( char* ) &m_nClass, sizeof ( int ) );
    f.read ( ( char* ) &m_nDomain, sizeof ( int ) );

    // set up NNs (only the last one is used)
    m_nn = new NN*[m_nCross+1];
    for ( int i=0;i<m_nCross+1;i++ )
        m_nn[i] = 0;
    m_nn[cross] = new NN();
    m_nn[cross]->setNrTargets ( m_nClass*m_nDomain );
    m_nn[cross]->setNrInputs ( m_nFeatures );
    m_nn[cross]->setNrExamplesTrain ( 0 );
    m_nn[cross]->setNrExamplesProbe ( 0 );
    m_nn[cross]->setTrainInputs ( 0 );
    m_nn[cross]->setTrainTargets ( 0 );
    m_nn[cross]->setProbeInputs ( 0 );
    m_nn[cross]->setProbeTargets ( 0 );

    // learn parameters
    m_nn[cross]->setInitWeightFactor ( m_initWeightFactor );
    m_nn[cross]->setLearnrate ( m_learnrate );
    m_nn[cross]->setLearnrateMinimum ( m_learnrateMinimum );
    m_nn[cross]->setLearnrateSubtractionValueAfterEverySample ( m_learnrateSubtractionValueAfterEverySample );
    m_nn[cross]->setLearnrateSubtractionValueAfterEveryEpoch ( m_learnrateSubtractionValueAfterEveryEpoch );
    m_nn[cross]->setMomentum ( m_momentum );
    m_nn[cross]->setWeightDecay ( m_weightDecay );
    m_nn[cross]->setMinUpdateErrorBound ( m_minUpdateErrorBound );
    m_nn[cross]->setBatchSize ( m_batchSize );
    m_nn[cross]->setMaxEpochs ( m_maxTuninigEpochs );
    m_nn[cross]->setL1Regularization ( m_enableL1Regularization );
    m_nn[cross]->enableErrorFunctionMAE ( m_enableErrorFunctionMAE );
    if(m_activationFunction=="tanh")
        m_nn[cross]->setActivationFunctionType(0);
    else if(m_activationFunction=="sin")
        m_nn[cross]->setActivationFunctionType(1);
    else if(m_activationFunction=="tanhMod0")
        m_nn[cross]->setActivationFunctionType(2);
    else
        assert(false);
    
    // set net inner stucture
    int nrLayer = m_nrLayer;
    int* neuronsPerLayer = Data::splitStringToIntegerList ( m_neuronsPerLayer, ',' );
    m_nn[cross]->setNNStructure ( nrLayer, neuronsPerLayer );

    m_nn[cross]->setRPROPPosNeg ( m_etaPosRPROP, m_etaNegRPROP );
    m_nn[cross]->setRPROPMinMaxUpdate ( m_minUpdateRPROP, m_maxUpdateRPROP );
    m_nn[cross]->setScaleOffset ( m_scaleOutputs, m_offsetOutputs );
    m_nn[cross]->setNormalTrainStopping ( true );
    m_nn[cross]->enableRPROP ( m_enableRPROP );
    m_nn[cross]->useBLASforTraining ( m_useBLASforTraining );
    m_nn[cross]->initNNWeights ( m_randSeed );
    delete[] neuronsPerLayer;

    int n = 0;
    f.read ( ( char* ) &n, sizeof ( int ) );

    REAL* w = new REAL[n];

    f.read ( ( char* ) w, sizeof ( REAL ) *n );
    f.read ( ( char* ) &m_maxSwing, sizeof ( double ) );
    f.close();

    // init the NN weights
    m_nn[cross]->setWeights ( w );

    if ( w )
        delete[] w;
    w = 0;
}

void NeuralNetwork::modelInit

(

)

[virtual]

Init the NN model

Implements StandardAlgorithm.

Definition at line 160 of file NeuralNetwork.cpp.

{
    // demonstration training on random data
    //demo();

    // add tunable parameters
    m_epoch = 0;
    paramEpochValues.push_back ( &m_epoch );
    paramEpochNames.push_back ( "epoch" );


    // set up NNs
    // nCross + 1 (for retraining)
    if ( m_nn == 0 )
    {
        m_nn = new NN*[m_nCross+1];
        for ( int i=0;i<m_nCross+1;i++ )
            m_nn[i] = 0;
    }
    for ( int i=0;i<m_nCross+1;i++ )
    {
        cout<<"Create a Neural Network ("<<i+1<<"/"<<m_nCross+1<<")"<<endl;
        if ( m_nn[i] == 0 )
            m_nn[i] = new NN();
        m_nn[i]->setNrTargets ( m_nClass*m_nDomain );
        m_nn[i]->setNrInputs ( m_nFeatures );
        m_nn[i]->setNrExamplesTrain ( 0 );
        m_nn[i]->setNrExamplesProbe ( 0 );
        m_nn[i]->setTrainInputs ( 0 );
        m_nn[i]->setTrainTargets ( 0 );
        m_nn[i]->setProbeInputs ( 0 );
        m_nn[i]->setProbeTargets ( 0 );
        m_nn[i]->setGlobalEpochs ( 0 );

        // learn parameters
        m_nn[i]->setInitWeightFactor ( m_initWeightFactor );
        m_nn[i]->setLearnrate ( m_learnrate );
        m_nn[i]->setLearnrateMinimum ( m_learnrateMinimum );
        m_nn[i]->setLearnrateSubtractionValueAfterEverySample ( m_learnrateSubtractionValueAfterEverySample );
        m_nn[i]->setLearnrateSubtractionValueAfterEveryEpoch ( m_learnrateSubtractionValueAfterEveryEpoch );
        m_nn[i]->setMomentum ( m_momentum );
        m_nn[i]->setWeightDecay ( m_weightDecay );
        m_nn[i]->setMinUpdateErrorBound ( m_minUpdateErrorBound );
        m_nn[i]->setBatchSize ( m_batchSize );
        m_nn[i]->setMaxEpochs ( m_maxTuninigEpochs );
        m_nn[i]->setL1Regularization ( m_enableL1Regularization );
        m_nn[i]->enableErrorFunctionMAE ( m_enableErrorFunctionMAE );
        if(m_activationFunction=="tanh")
            m_nn[i]->setActivationFunctionType(0);
        else if(m_activationFunction=="sin")
            m_nn[i]->setActivationFunctionType(1);
        else if(m_activationFunction=="tanhMod0")
            m_nn[i]->setActivationFunctionType(2);
        else
            assert(false);
        // set net inner stucture
        int nrLayer = m_nrLayer;
        int* neuronsPerLayer = Data::splitStringToIntegerList ( m_neuronsPerLayer, ',' );
        m_nn[i]->enableRPROP ( m_enableRPROP );
        m_nn[i]->setNNStructure ( nrLayer, neuronsPerLayer );

        m_nn[i]->setScaleOffset ( m_scaleOutputs, m_offsetOutputs );
        m_nn[i]->setRPROPPosNeg ( m_etaPosRPROP, m_etaNegRPROP );
        m_nn[i]->setRPROPMinMaxUpdate ( m_minUpdateRPROP, m_maxUpdateRPROP );
        m_nn[i]->setNormalTrainStopping ( true );
        m_nn[i]->useBLASforTraining ( m_useBLASforTraining );
        m_nn[i]->initNNWeights ( m_randSeed );
        delete[] neuronsPerLayer;

        cout<<endl<<endl;
    }
}

void NeuralNetwork::modelUpdate	(	REAL *	input,
		REAL *	target,
		uint	nSamples,
		uint	crossRun
	)			[virtual]

Make a model update, set the new cross validation set or set the whole training set for retraining

Parameters:

	input	Pointer to input (can be cross validation set, or whole training set) (rows x nFeatures)
	target	The targets (can be cross validation targets)
	nSamples	The sample size (rows) in input
	crossRun	The cross validation run (for training)

Implements StandardAlgorithm.

Definition at line 257 of file NeuralNetwork.cpp.

{
    m_nn[crossRun]->setTrainInputs ( input );
    m_nn[crossRun]->setTrainTargets ( target );
    m_nn[crossRun]->setNrExamplesTrain ( nSamples );

    if ( crossRun < m_nCross || (crossRun==0 && m_validationType=="ValidationSet") )
    {
        // one gradient descent step (one epoch)
        m_nn[crossRun]->trainOneEpoch();

        stringstream s;
        s<<"[t:"<<sqrt ( m_nn[crossRun]->m_sumSquaredError/ ( double ) m_nn[crossRun]->m_sumSquaredErrorSamples ) <<"]";
        cout<<s.str();
        if ( crossRun == m_nCross - 1 || (crossRun==0 && m_validationType=="ValidationSet") )
            m_nn[crossRun]->printLearnrate();
    }
    else
    {
        // retraining with fix number of epochs
        cout<<endl<<"Tune: Training of full trainset "<<endl;
        m_nn[crossRun]->setNormalTrainStopping ( false );
        int maxEpochs = m_epochParamBest[0];
        if ( maxEpochs == 0 )
            maxEpochs = 1;  // train at least one epoch
        cout<<"Best #epochs (on cross validation): "<<maxEpochs<<endl;
        m_nn[crossRun]->setMaxEpochs ( maxEpochs );

        // train the net
        int epochs = m_nn[crossRun]->trainNN();
        cout<<endl;
    }
}

void NeuralNetwork::predictAllOutputs	(	REAL *	rawInputs,
		REAL *	outputs,
		uint	nSamples,
		uint	crossRun
	)			[virtual]

Prediction for outside use, predicts outputs based on raw input values

Parameters:

	rawInputs	The input feature, without normalization (raw)
	outputs	The output value (prediction of target)
	nSamples	The input size (number of rows)
	crossRun	Number of cross validation run (in training)

Implements StandardAlgorithm.

Definition at line 241 of file NeuralNetwork.cpp.

{
    // predict all samples
    for ( int i=0;i<nSamples;i++ )
        m_nn[crossRun]->predictSingleInput ( rawInputs + i*m_nFeatures, outputs + i*m_nClass*m_nDomain );
}

void NeuralNetwork::readSpecificMaps

(

)

[virtual]

Read the Algorithm specific values from the description file

Implements StandardAlgorithm.

Definition at line 58 of file NeuralNetwork.cpp.

{
    cout<<"Read specific maps"<<endl;

    // read dsc vars
    m_nrLayer = m_intMap["nrLayer"];
    m_batchSize = m_intMap["batchSize"];
    m_offsetOutputs = m_doubleMap["offsetOutputs"];
    m_scaleOutputs = m_doubleMap["scaleOutputs"];
    m_initWeightFactor = m_doubleMap["initWeightFactor"];
    m_learnrate = m_doubleMap["learnrate"];
    m_learnrateMinimum = m_doubleMap["learnrateMinimum"];
    m_learnrateSubtractionValueAfterEverySample = m_doubleMap["learnrateSubtractionValueAfterEverySample"];
    m_learnrateSubtractionValueAfterEveryEpoch = m_doubleMap["learnrateSubtractionValueAfterEveryEpoch"];
    m_momentum = m_doubleMap["momentum"];
    m_weightDecay = m_doubleMap["weightDecay"];
    m_minUpdateErrorBound = m_doubleMap["minUpdateErrorBound"];
    m_etaPosRPROP = m_doubleMap["etaPosRPROP"];
    m_etaNegRPROP = m_doubleMap["etaNegRPROP"];
    m_minUpdateRPROP = m_doubleMap["minUpdateRPROP"];
    m_maxUpdateRPROP = m_doubleMap["maxUpdateRPROP"];
    m_enableL1Regularization = m_boolMap["enableL1Regularization"];
    m_enableErrorFunctionMAE = m_boolMap["enableErrorFunctionMAE"];
    m_enableRPROP = m_boolMap["enableRPROP"];
    m_useBLASforTraining = m_boolMap["useBLASforTraining"];
    m_neuronsPerLayer = m_stringMap["neuronsPerLayer"];
    m_activationFunction = m_stringMap["activationFunction"];
}

void NeuralNetwork::saveWeights

(

int

cross

)

[virtual]

Save the weights and all other parameters for load the complete prediction model

Implements StandardAlgorithm.

Definition at line 295 of file NeuralNetwork.cpp.

{
    char buf[1024];
    sprintf ( buf,"%02d",cross );
    string name = m_datasetPath + "/" + m_tempPath + "/" + m_weightFile + "." + buf;
    if ( m_inRetraining )
        cout<<"Save:"<<name<<endl;
    int n = m_nn[cross]->getNrWeights();
    REAL* w = m_nn[cross]->getWeightPtr();

    fstream f ( name.c_str(), ios::out );
    f.write ( ( char* ) &m_nTrain, sizeof ( int ) );
    f.write ( ( char* ) &m_nFeatures, sizeof ( int ) );
    f.write ( ( char* ) &m_nClass, sizeof ( int ) );
    f.write ( ( char* ) &m_nDomain, sizeof ( int ) );
    f.write ( ( char* ) &n, sizeof ( int ) );
    f.write ( ( char* ) w, sizeof ( REAL ) *n );
    f.write ( ( char* ) &m_maxSwing, sizeof ( double ) );
    f.close();
}

string NeuralNetwork::templateGenerator	(	int	id,
		string	preEffect,
		int	nameID,
		bool	blendStop
	)			[static]

Generates a template of the description file

Returns:

The template string

Definition at line 415 of file NeuralNetwork.cpp.

{
    stringstream s;
    s<<"ALGORITHM=NeuralNetwork"<<endl;
    s<<"ID="<<id<<endl;
    s<<"TRAIN_ON_FULLPREDICTOR="<<preEffect<<endl;
    s<<"DISABLE=0"<<endl;
    s<<endl;
    s<<"[int]"<<endl;
    s<<"nrLayer=3"<<endl;
    s<<"batchSize=1"<<endl;
    s<<"minTuninigEpochs=30"<<endl;
    s<<"maxTuninigEpochs=100"<<endl;
    s<<endl;
    s<<"[double]"<<endl;
    s<<"initMaxSwing=1.0"<<endl;
    s<<endl;
    s<<"offsetOutputs=0.0"<<endl;
    s<<"scaleOutputs=1.2"<<endl;
    s<<endl;
    s<<"etaPosRPROP=1.005"<<endl;
    s<<"etaNegRPROP=0.99"<<endl;
    s<<"minUpdateRPROP=1e-8"<<endl;
    s<<"maxUpdateRPROP=1e-2"<<endl;
    s<<endl;
    s<<"initWeightFactor=1.0"<<endl;
    s<<"learnrate=1e-3"<<endl;
    s<<"learnrateMinimum=1e-5"<<endl;
    s<<"learnrateSubtractionValueAfterEverySample=0.0"<<endl;
    s<<"learnrateSubtractionValueAfterEveryEpoch=0.0"<<endl;
    s<<"momentum=0.0"<<endl;
    s<<"weightDecay=0.0"<<endl;
    s<<"minUpdateErrorBound=1e-6"<<endl;
    s<<endl;
    s<<"[bool]"<<endl;
    s<<"enableErrorFunctionMAE=0"<<endl;
    s<<"enableL1Regularization=0"<<endl;
    s<<"enableClipping=1"<<endl;
    s<<"enableTuneSwing=0"<<endl;
    s<<"useBLASforTraining=1"<<endl;
    s<<"enableRPROP=0"<<endl;
    s<<endl;
    s<<"minimzeProbe="<< ( !blendStop ) <<endl;
    s<<"minimzeProbeClassificationError=0"<<endl;
    s<<"minimzeBlend="<<blendStop<<endl;
    s<<"minimzeBlendClassificationError=0"<<endl;
    s<<endl;
    s<<"[string]"<<endl;
    s<<"activationFunction=tanh"<<endl;
    s<<"neuronsPerLayer=30,20,40,30,100,-1"<<endl;
    s<<"weightFile=NeuralNetwork_"<<nameID<<"_weights.dat"<<endl;
    s<<"fullPrediction=NeuralNetwork_"<<nameID<<".dat"<<endl;

    return s.str();
}

---

The documentation for this class was generated from the following files:

• ELF/NeuralNetwork.h
• ELF/NeuralNetwork.cpp