NeuralNetwork.cpp

#include "NeuralNetwork.h"

extern StreamOutput cout;

NeuralNetwork::NeuralNetwork()
{
    cout<<"NeuralNetwork"<<endl;
    // init member vars
    m_inputs = 0;
    m_nn = 0;
    m_epoch = 0;
    m_nrLayer = 0;
    m_batchSize = 0;
    m_offsetOutputs = 0;
    m_scaleOutputs = 0;
    m_initWeightFactor = 0;
    m_learnrate = 0;
    m_learnrateMinimum = 0;
    m_learnrateSubtractionValueAfterEverySample = 0;
    m_learnrateSubtractionValueAfterEveryEpoch = 0;
    m_momentum = 0;
    m_weightDecay = 0;
    m_minUpdateErrorBound = 0;
    m_etaPosRPROP = 0;
    m_etaNegRPROP = 0;
    m_minUpdateRPROP = 0;
    m_maxUpdateRPROP = 0;
    m_enableRPROP = 0;
    m_useBLASforTraining = 0;
    m_enableL1Regularization = 0;
    m_enableErrorFunctionMAE = 0;
}

NeuralNetwork::~NeuralNetwork()
{
    cout<<"descructor NeuralNetwork"<<endl;

    for ( int i=0;i<m_nCross+1;i++ )
    {
        if ( m_nn[i] )
            delete m_nn[i];
        m_nn[i] = 0;
    }
    delete[] m_nn;

}

void NeuralNetwork::readSpecificMaps()
{
    cout<<"Read specific maps"<<endl;

    // read dsc vars
    m_nrLayer = m_intMap["nrLayer"];
    m_batchSize = m_intMap["batchSize"];
    m_offsetOutputs = m_doubleMap["offsetOutputs"];
    m_scaleOutputs = m_doubleMap["scaleOutputs"];
    m_initWeightFactor = m_doubleMap["initWeightFactor"];
    m_learnrate = m_doubleMap["learnrate"];
    m_learnrateMinimum = m_doubleMap["learnrateMinimum"];
    m_learnrateSubtractionValueAfterEverySample = m_doubleMap["learnrateSubtractionValueAfterEverySample"];
    m_learnrateSubtractionValueAfterEveryEpoch = m_doubleMap["learnrateSubtractionValueAfterEveryEpoch"];
    m_momentum = m_doubleMap["momentum"];
    m_weightDecay = m_doubleMap["weightDecay"];
    m_minUpdateErrorBound = m_doubleMap["minUpdateErrorBound"];
    m_etaPosRPROP = m_doubleMap["etaPosRPROP"];
    m_etaNegRPROP = m_doubleMap["etaNegRPROP"];
    m_minUpdateRPROP = m_doubleMap["minUpdateRPROP"];
    m_maxUpdateRPROP = m_doubleMap["maxUpdateRPROP"];
    m_enableL1Regularization = m_boolMap["enableL1Regularization"];
    m_enableErrorFunctionMAE = m_boolMap["enableErrorFunctionMAE"];
    m_enableRPROP = m_boolMap["enableRPROP"];
    m_useBLASforTraining = m_boolMap["useBLASforTraining"];
    m_neuronsPerLayer = m_stringMap["neuronsPerLayer"];
    m_activationFunction = m_stringMap["activationFunction"];
}

void NeuralNetwork::demo()
{
    int inputs = 784, targets = 10, exampleTrain = 60000, exampleTest = 100;
    REAL* trainInput = new REAL[inputs * exampleTrain];
    REAL* testInput = new REAL[inputs * exampleTest];
    REAL* trainTarget = new REAL[targets * exampleTrain];
    REAL* testTarget = new REAL[targets * exampleTest];

    for ( int i=0;i<inputs*exampleTrain;i++ )
        trainInput[i] = ( double ) rand() / ( double ) RAND_MAX - 0.5;
    for ( int i=0;i<inputs*exampleTest;i++ )
        testInput[i] = ( double ) rand() / ( double ) RAND_MAX - 0.5;
    for ( int i=0;i<targets*exampleTrain;i++ )
        trainTarget[i] = ( double ) rand() / ( double ) RAND_MAX - 0.5;
    for ( int i=0;i<targets*exampleTest;i++ )
        testTarget[i] = ( double ) rand() / ( double ) RAND_MAX - 0.5;

    NN nn;
    nn.setNrTargets ( targets );
    nn.setNrInputs ( inputs );
    nn.setNrExamplesTrain ( exampleTrain );
    nn.setNrExamplesProbe ( exampleTest );
    nn.setTrainInputs ( trainInput );
    nn.setTrainTargets ( trainTarget );
    nn.setProbeInputs ( testInput );
    nn.setProbeTargets ( testTarget );

    // learn parameters
    nn.setInitWeightFactor ( m_initWeightFactor );
    nn.setLearnrate ( m_learnrate );
    nn.setLearnrateMinimum ( m_learnrateMinimum );
    nn.setLearnrateSubtractionValueAfterEverySample ( m_learnrateSubtractionValueAfterEverySample );
    nn.setLearnrateSubtractionValueAfterEveryEpoch ( m_learnrateSubtractionValueAfterEveryEpoch );
    nn.setMomentum ( m_momentum );
    nn.setWeightDecay ( m_weightDecay );
    nn.setMinUpdateErrorBound ( m_minUpdateErrorBound );
    nn.setBatchSize ( m_batchSize );
    nn.setMaxEpochs ( m_maxTuninigEpochs );
    nn.setL1Regularization ( m_enableL1Regularization );
    nn.enableErrorFunctionMAE ( m_enableErrorFunctionMAE );

    // set net inner stucture
    int nrLayer = m_nrLayer;
    int* neuronsPerLayer = Data::splitStringToIntegerList ( m_neuronsPerLayer, ',' );
    nn.enableRPROP ( m_enableRPROP );
    nn.setNNStructure ( nrLayer, neuronsPerLayer );
    nn.useBLASforTraining ( m_useBLASforTraining );
    nn.setScaleOffset ( 1.0, 0.0 );
    nn.setRPROPPosNeg ( m_etaPosRPROP, m_etaNegRPROP );
    nn.setRPROPMinMaxUpdate ( m_minUpdateRPROP, m_maxUpdateRPROP );
    nn.setNormalTrainStopping ( true );
    nn.initNNWeights ( 0 );
    nn.setActivationFunctionType( 0 );
    delete[] neuronsPerLayer;

    // training
    nn.trainNN();

    delete[] trainInput;
    delete[] testInput;
    delete[] trainTarget;
    delete[] testTarget;
}

void NeuralNetwork::modelInit()
{
    // demonstration training on random data
    //demo();

    // add tunable parameters
    m_epoch = 0;
    paramEpochValues.push_back ( &m_epoch );
    paramEpochNames.push_back ( "epoch" );


    // set up NNs
    // nCross + 1 (for retraining)
    if ( m_nn == 0 )
    {
        m_nn = new NN*[m_nCross+1];
        for ( int i=0;i<m_nCross+1;i++ )
            m_nn[i] = 0;
    }
    for ( int i=0;i<m_nCross+1;i++ )
    {
        cout<<"Create a Neural Network ("<<i+1<<"/"<<m_nCross+1<<")"<<endl;
        if ( m_nn[i] == 0 )
            m_nn[i] = new NN();
        m_nn[i]->setNrTargets ( m_nClass*m_nDomain );
        m_nn[i]->setNrInputs ( m_nFeatures );
        m_nn[i]->setNrExamplesTrain ( 0 );
        m_nn[i]->setNrExamplesProbe ( 0 );
        m_nn[i]->setTrainInputs ( 0 );
        m_nn[i]->setTrainTargets ( 0 );
        m_nn[i]->setProbeInputs ( 0 );
        m_nn[i]->setProbeTargets ( 0 );
        m_nn[i]->setGlobalEpochs ( 0 );

        // learn parameters
        m_nn[i]->setInitWeightFactor ( m_initWeightFactor );
        m_nn[i]->setLearnrate ( m_learnrate );
        m_nn[i]->setLearnrateMinimum ( m_learnrateMinimum );
        m_nn[i]->setLearnrateSubtractionValueAfterEverySample ( m_learnrateSubtractionValueAfterEverySample );
        m_nn[i]->setLearnrateSubtractionValueAfterEveryEpoch ( m_learnrateSubtractionValueAfterEveryEpoch );
        m_nn[i]->setMomentum ( m_momentum );
        m_nn[i]->setWeightDecay ( m_weightDecay );
        m_nn[i]->setMinUpdateErrorBound ( m_minUpdateErrorBound );
        m_nn[i]->setBatchSize ( m_batchSize );
        m_nn[i]->setMaxEpochs ( m_maxTuninigEpochs );
        m_nn[i]->setL1Regularization ( m_enableL1Regularization );
        m_nn[i]->enableErrorFunctionMAE ( m_enableErrorFunctionMAE );
        if(m_activationFunction=="tanh")
            m_nn[i]->setActivationFunctionType(0);
        else if(m_activationFunction=="sin")
            m_nn[i]->setActivationFunctionType(1);
        else if(m_activationFunction=="tanhMod0")
            m_nn[i]->setActivationFunctionType(2);
        else
            assert(false);
        // set net inner stucture
        int nrLayer = m_nrLayer;
        int* neuronsPerLayer = Data::splitStringToIntegerList ( m_neuronsPerLayer, ',' );
        m_nn[i]->enableRPROP ( m_enableRPROP );
        m_nn[i]->setNNStructure ( nrLayer, neuronsPerLayer );

        m_nn[i]->setScaleOffset ( m_scaleOutputs, m_offsetOutputs );
        m_nn[i]->setRPROPPosNeg ( m_etaPosRPROP, m_etaNegRPROP );
        m_nn[i]->setRPROPMinMaxUpdate ( m_minUpdateRPROP, m_maxUpdateRPROP );
        m_nn[i]->setNormalTrainStopping ( true );
        m_nn[i]->useBLASforTraining ( m_useBLASforTraining );
        m_nn[i]->initNNWeights ( m_randSeed );
        delete[] neuronsPerLayer;

        cout<<endl<<endl;
    }
}

void NeuralNetwork::predictAllOutputs ( REAL* rawInputs, REAL* outputs, uint nSamples, uint crossRun )
{
    // predict all samples
    for ( int i=0;i<nSamples;i++ )
        m_nn[crossRun]->predictSingleInput ( rawInputs + i*m_nFeatures, outputs + i*m_nClass*m_nDomain );
}

void NeuralNetwork::modelUpdate ( REAL* input, REAL* target, uint nSamples, uint crossRun )
{
    m_nn[crossRun]->setTrainInputs ( input );
    m_nn[crossRun]->setTrainTargets ( target );
    m_nn[crossRun]->setNrExamplesTrain ( nSamples );

    if ( crossRun < m_nCross || (crossRun==0 && m_validationType=="ValidationSet") )
    {
        // one gradient descent step (one epoch)
        m_nn[crossRun]->trainOneEpoch();

        stringstream s;
        s<<"[t:"<<sqrt ( m_nn[crossRun]->m_sumSquaredError/ ( double ) m_nn[crossRun]->m_sumSquaredErrorSamples ) <<"]";
        cout<<s.str();
        if ( crossRun == m_nCross - 1 || (crossRun==0 && m_validationType=="ValidationSet") )
            m_nn[crossRun]->printLearnrate();
    }
    else
    {
        // retraining with fix number of epochs
        cout<<endl<<"Tune: Training of full trainset "<<endl;
        m_nn[crossRun]->setNormalTrainStopping ( false );
        int maxEpochs = m_epochParamBest[0];
        if ( maxEpochs == 0 )
            maxEpochs = 1;  // train at least one epoch
        cout<<"Best #epochs (on cross validation): "<<maxEpochs<<endl;
        m_nn[crossRun]->setMaxEpochs ( maxEpochs );

        // train the net
        int epochs = m_nn[crossRun]->trainNN();
        cout<<endl;
    }
}

void NeuralNetwork::saveWeights ( int cross )
{
    char buf[1024];
    sprintf ( buf,"%02d",cross );
    string name = m_datasetPath + "/" + m_tempPath + "/" + m_weightFile + "." + buf;
    if ( m_inRetraining )
        cout<<"Save:"<<name<<endl;
    int n = m_nn[cross]->getNrWeights();
    REAL* w = m_nn[cross]->getWeightPtr();

    fstream f ( name.c_str(), ios::out );
    f.write ( ( char* ) &m_nTrain, sizeof ( int ) );
    f.write ( ( char* ) &m_nFeatures, sizeof ( int ) );
    f.write ( ( char* ) &m_nClass, sizeof ( int ) );
    f.write ( ( char* ) &m_nDomain, sizeof ( int ) );
    f.write ( ( char* ) &n, sizeof ( int ) );
    f.write ( ( char* ) w, sizeof ( REAL ) *n );
    f.write ( ( char* ) &m_maxSwing, sizeof ( double ) );
    f.close();
}

void NeuralNetwork::loadWeights ( int cross )
{
    // load weights
    char buf[1024];
    sprintf ( buf,"%02d",cross );
    string name = m_datasetPath + "/" + m_tempPath + "/" + m_weightFile + "." + buf;
    cout<<"Load:"<<name<<endl;
    fstream f ( name.c_str(), ios::in );
    if ( f.is_open() == false )
        assert ( false );
    f.read ( ( char* ) &m_nTrain, sizeof ( int ) );
    f.read ( ( char* ) &m_nFeatures, sizeof ( int ) );
    f.read ( ( char* ) &m_nClass, sizeof ( int ) );
    f.read ( ( char* ) &m_nDomain, sizeof ( int ) );

    // set up NNs (only the last one is used)
    m_nn = new NN*[m_nCross+1];
    for ( int i=0;i<m_nCross+1;i++ )
        m_nn[i] = 0;
    m_nn[cross] = new NN();
    m_nn[cross]->setNrTargets ( m_nClass*m_nDomain );
    m_nn[cross]->setNrInputs ( m_nFeatures );
    m_nn[cross]->setNrExamplesTrain ( 0 );
    m_nn[cross]->setNrExamplesProbe ( 0 );
    m_nn[cross]->setTrainInputs ( 0 );
    m_nn[cross]->setTrainTargets ( 0 );
    m_nn[cross]->setProbeInputs ( 0 );
    m_nn[cross]->setProbeTargets ( 0 );

    // learn parameters
    m_nn[cross]->setInitWeightFactor ( m_initWeightFactor );
    m_nn[cross]->setLearnrate ( m_learnrate );
    m_nn[cross]->setLearnrateMinimum ( m_learnrateMinimum );
    m_nn[cross]->setLearnrateSubtractionValueAfterEverySample ( m_learnrateSubtractionValueAfterEverySample );
    m_nn[cross]->setLearnrateSubtractionValueAfterEveryEpoch ( m_learnrateSubtractionValueAfterEveryEpoch );
    m_nn[cross]->setMomentum ( m_momentum );
    m_nn[cross]->setWeightDecay ( m_weightDecay );
    m_nn[cross]->setMinUpdateErrorBound ( m_minUpdateErrorBound );
    m_nn[cross]->setBatchSize ( m_batchSize );
    m_nn[cross]->setMaxEpochs ( m_maxTuninigEpochs );
    m_nn[cross]->setL1Regularization ( m_enableL1Regularization );
    m_nn[cross]->enableErrorFunctionMAE ( m_enableErrorFunctionMAE );
    if(m_activationFunction=="tanh")
        m_nn[cross]->setActivationFunctionType(0);
    else if(m_activationFunction=="sin")
        m_nn[cross]->setActivationFunctionType(1);
    else if(m_activationFunction=="tanhMod0")
        m_nn[cross]->setActivationFunctionType(2);
    else
        assert(false);
    
    // set net inner stucture
    int nrLayer = m_nrLayer;
    int* neuronsPerLayer = Data::splitStringToIntegerList ( m_neuronsPerLayer, ',' );
    m_nn[cross]->setNNStructure ( nrLayer, neuronsPerLayer );

    m_nn[cross]->setRPROPPosNeg ( m_etaPosRPROP, m_etaNegRPROP );
    m_nn[cross]->setRPROPMinMaxUpdate ( m_minUpdateRPROP, m_maxUpdateRPROP );
    m_nn[cross]->setScaleOffset ( m_scaleOutputs, m_offsetOutputs );
    m_nn[cross]->setNormalTrainStopping ( true );
    m_nn[cross]->enableRPROP ( m_enableRPROP );
    m_nn[cross]->useBLASforTraining ( m_useBLASforTraining );
    m_nn[cross]->initNNWeights ( m_randSeed );
    delete[] neuronsPerLayer;

    int n = 0;
    f.read ( ( char* ) &n, sizeof ( int ) );

    REAL* w = new REAL[n];

    f.read ( ( char* ) w, sizeof ( REAL ) *n );
    f.read ( ( char* ) &m_maxSwing, sizeof ( double ) );
    f.close();

    // init the NN weights
    m_nn[cross]->setWeights ( w );

    if ( w )
        delete[] w;
    w = 0;
}

void NeuralNetwork::loadMetaWeights ( int cross )
{
    // nothing to do in a gradient-descent based algorithm
}

string NeuralNetwork::templateGenerator ( int id, string preEffect, int nameID, bool blendStop )
{
    stringstream s;
    s<<"ALGORITHM=NeuralNetwork"<<endl;
    s<<"ID="<<id<<endl;
    s<<"TRAIN_ON_FULLPREDICTOR="<<preEffect<<endl;
    s<<"DISABLE=0"<<endl;
    s<<endl;
    s<<"[int]"<<endl;
    s<<"nrLayer=3"<<endl;
    s<<"batchSize=1"<<endl;
    s<<"minTuninigEpochs=30"<<endl;
    s<<"maxTuninigEpochs=100"<<endl;
    s<<endl;
    s<<"[double]"<<endl;
    s<<"initMaxSwing=1.0"<<endl;
    s<<endl;
    s<<"offsetOutputs=0.0"<<endl;
    s<<"scaleOutputs=1.2"<<endl;
    s<<endl;
    s<<"etaPosRPROP=1.005"<<endl;
    s<<"etaNegRPROP=0.99"<<endl;
    s<<"minUpdateRPROP=1e-8"<<endl;
    s<<"maxUpdateRPROP=1e-2"<<endl;
    s<<endl;
    s<<"initWeightFactor=1.0"<<endl;
    s<<"learnrate=1e-3"<<endl;
    s<<"learnrateMinimum=1e-5"<<endl;
    s<<"learnrateSubtractionValueAfterEverySample=0.0"<<endl;
    s<<"learnrateSubtractionValueAfterEveryEpoch=0.0"<<endl;
    s<<"momentum=0.0"<<endl;
    s<<"weightDecay=0.0"<<endl;
    s<<"minUpdateErrorBound=1e-6"<<endl;
    s<<endl;
    s<<"[bool]"<<endl;
    s<<"enableErrorFunctionMAE=0"<<endl;
    s<<"enableL1Regularization=0"<<endl;
    s<<"enableClipping=1"<<endl;
    s<<"enableTuneSwing=0"<<endl;
    s<<"useBLASforTraining=1"<<endl;
    s<<"enableRPROP=0"<<endl;
    s<<endl;
    s<<"minimzeProbe="<< ( !blendStop ) <<endl;
    s<<"minimzeProbeClassificationError=0"<<endl;
    s<<"minimzeBlend="<<blendStop<<endl;
    s<<"minimzeBlendClassificationError=0"<<endl;
    s<<endl;
    s<<"[string]"<<endl;
    s<<"activationFunction=tanh"<<endl;
    s<<"neuronsPerLayer=30,20,40,30,100,-1"<<endl;
    s<<"weightFile=NeuralNetwork_"<<nameID<<"_weights.dat"<<endl;
    s<<"fullPrediction=NeuralNetwork_"<<nameID<<".dat"<<endl;

    return s.str();
}