NeuralNetworkRBMauto.cpp

#include "NeuralNetworkRBMauto.h"

extern StreamOutput cout;

NeuralNetworkRBMauto::NeuralNetworkRBMauto()
{
    cout<<"NeuralNetworkRBMauto"<<endl;
    // init member vars
    m_inputs = 0;
    m_nn = 0;
    m_epoch = 0;
    m_nrLayerAuto = 0;
    m_nrLayerNetHidden = 0;
    m_batchSize = 0;
    m_offsetOutputs = 0;
    m_scaleOutputs = 0;
    m_initWeightFactor = 0;
    m_learnrate = 0;
    m_learnrateMinimum = 0;
    m_learnrateSubtractionValueAfterEverySample = 0;
    m_learnrateSubtractionValueAfterEveryEpoch = 0;
    m_momentum = 0;
    m_weightDecay = 0;
    m_minUpdateErrorBound = 0;
    m_etaPosRPROP = 0;
    m_etaNegRPROP = 0;
    m_minUpdateRPROP = 0;
    m_maxUpdateRPROP = 0;
    m_enableRPROP = 0;
    m_useBLASforTraining = 0;
    m_enableL1Regularization = 0;
    m_enableErrorFunctionMAE = 0;
    m_isFirstEpoch = 0;
    m_tuningEpochsRBMFinetuning = 0;
    m_tuningEpochsOutputNet = 0;
    m_learnrateAutoencoderFinetuning = 0;
}

NeuralNetworkRBMauto::~NeuralNetworkRBMauto()
{
    cout<<"descructor NeuralNetworkRBMauto"<<endl;

    for ( int i=0;i<m_nCross+1;i++ )
    {
        if ( m_nn[i] )
            delete m_nn[i];
        m_nn[i] = 0;
    }
    delete[] m_nn;

    if ( m_isFirstEpoch )
        delete[] m_isFirstEpoch;
    m_isFirstEpoch = 0;
}

void NeuralNetworkRBMauto::readSpecificMaps()
{
    cout<<"Read specific maps"<<endl;

    // read dsc vars
    m_nrLayerAuto = m_intMap["nrLayerAuto"];
    m_nrLayerNetHidden = m_intMap["nrLayerNetHidden"];

    m_batchSize = m_intMap["batchSize"];
    m_tuningEpochsRBMFinetuning = m_intMap["tuningEpochsRBMFinetuning"];
    m_tuningEpochsOutputNet = m_intMap["tuningEpochsOutputNet"];
    m_offsetOutputs = m_doubleMap["offsetOutputs"];
    m_scaleOutputs = m_doubleMap["scaleOutputs"];
    m_initWeightFactor = m_doubleMap["initWeightFactor"];
    m_learnrate = m_doubleMap["learnrate"];
    m_learnrateAutoencoderFinetuning = m_doubleMap["learnrateAutoencoderFinetuning"];
    m_learnrateMinimum = m_doubleMap["learnrateMinimum"];
    m_learnrateSubtractionValueAfterEverySample = m_doubleMap["learnrateSubtractionValueAfterEverySample"];
    m_learnrateSubtractionValueAfterEveryEpoch = m_doubleMap["learnrateSubtractionValueAfterEveryEpoch"];
    m_momentum = m_doubleMap["momentum"];
    m_weightDecay = m_doubleMap["weightDecay"];
    m_minUpdateErrorBound = m_doubleMap["minUpdateErrorBound"];
    m_etaPosRPROP = m_doubleMap["etaPosRPROP"];
    m_etaNegRPROP = m_doubleMap["etaNegRPROP"];
    m_minUpdateRPROP = m_doubleMap["minUpdateRPROP"];
    m_maxUpdateRPROP = m_doubleMap["maxUpdateRPROP"];
    m_enableL1Regularization = m_boolMap["enableL1Regularization"];
    m_enableErrorFunctionMAE = m_boolMap["enableErrorFunctionMAE"];
    m_enableRPROP = m_boolMap["enableRPROP"];
    m_useBLASforTraining = m_boolMap["useBLASforTraining"];
    m_neuronsPerLayerAuto = m_stringMap["neuronsPerLayerAuto"];
    m_neuronsPerLayerNetHidden = m_stringMap["neuronsPerLayerNetHidden"];
}

void NeuralNetworkRBMauto::modelInit()
{
    // add tunable parameters
    m_epoch = 0;
    paramEpochValues.push_back ( &m_epoch );
    paramEpochNames.push_back ( "epoch" );

    // set up NNs
    // nCross + 1 (for retraining)
    if ( m_nn == 0 )
    {
        m_nn = new NNRBM*[m_nCross+1];
        for ( int i=0;i<m_nCross+1;i++ )
            m_nn[i] = 0;
    }
    for ( int i=0;i<m_nCross+1;i++ )
    {
        cout<<"Create a Neural Network ("<<i+1<<"/"<<m_nCross+1<<")"<<endl;
        if ( m_nn[i] == 0 )
            m_nn[i] = new NNRBM();
        m_nn[i]->setNrTargets ( m_nClass*m_nDomain );
        m_nn[i]->setNrInputs ( m_nFeatures );
        m_nn[i]->setNrExamplesTrain ( 0 );
        m_nn[i]->setNrExamplesProbe ( 0 );
        m_nn[i]->setTrainInputs ( 0 );
        m_nn[i]->setTrainTargets ( 0 );
        m_nn[i]->setProbeInputs ( 0 );
        m_nn[i]->setProbeTargets ( 0 );
        m_nn[i]->setGlobalEpochs ( 0 );

        // learn parameters
        m_nn[i]->setInitWeightFactor ( m_initWeightFactor );
        m_nn[i]->setLearnrate ( m_learnrate );
        m_nn[i]->setLearnrateMinimum ( m_learnrateMinimum );
        m_nn[i]->setLearnrateSubtractionValueAfterEverySample ( m_learnrateSubtractionValueAfterEverySample );
        m_nn[i]->setLearnrateSubtractionValueAfterEveryEpoch ( m_learnrateSubtractionValueAfterEveryEpoch );
        m_nn[i]->setMomentum ( m_momentum );
        m_nn[i]->setWeightDecay ( m_weightDecay );
        m_nn[i]->setMinUpdateErrorBound ( m_minUpdateErrorBound );
        m_nn[i]->setBatchSize ( m_batchSize );
        m_nn[i]->setMaxEpochs ( m_maxTuninigEpochs );
        m_nn[i]->setL1Regularization ( m_enableL1Regularization );
        m_nn[i]->enableErrorFunctionMAE ( m_enableErrorFunctionMAE );

        m_nn[i]->setRBMLearnParams ( m_doubleMap["rbmLearnrateWeights"], m_doubleMap["rbmLearnrateBiasVis"], m_doubleMap["rbmLearnrateBiasHid"], m_doubleMap["rbmWeightDecay"], m_doubleMap["rbmMaxEpochs"] );

        // set net inner stucture
        int nrLayer = m_nrLayerAuto + m_nrLayerNetHidden;
        int* neuronsPerLayerAuto = Data::splitStringToIntegerList ( m_neuronsPerLayerAuto, ',' );
        int* neuronsPerLayerNetHidden = Data::splitStringToIntegerList ( m_neuronsPerLayerNetHidden, ',' );
        int* neurPerLayer = new int[nrLayer+1];
        int* layerType = new int[nrLayer+2];
        for ( int j=0;j<m_nrLayerAuto;j++ )
        {
            neurPerLayer[j] = neuronsPerLayerAuto[j];
            layerType[j] = 0;  // sig:0, linear:1, tanh:2, softmax:3
        }
        layerType[m_nrLayerAuto] = 1;  // sig:0, linear:1, tanh:2, softmax:3
        for ( int j=0;j<m_nrLayerNetHidden;j++ )
        {
            neurPerLayer[j+m_nrLayerAuto] = neuronsPerLayerNetHidden[j];
            layerType[j+m_nrLayerAuto+1] = 2;  // sig:0, linear:1, tanh:2, softmax:3
        }
        layerType[m_nrLayerNetHidden+m_nrLayerAuto+1] = 2;  // sig:0, linear:1, tanh:2, softmax:3

        m_nn[i]->enableRPROP ( m_enableRPROP );
        m_nn[i]->setNNStructure ( nrLayer+1, neurPerLayer, false, layerType );

        m_nn[i]->setScaleOffset ( m_scaleOutputs, m_offsetOutputs );
        m_nn[i]->setRPROPPosNeg ( m_etaPosRPROP, m_etaNegRPROP );
        m_nn[i]->setRPROPMinMaxUpdate ( m_minUpdateRPROP, m_maxUpdateRPROP );
        m_nn[i]->setNormalTrainStopping ( true );
        m_nn[i]->useBLASforTraining ( m_useBLASforTraining );
        m_nn[i]->initNNWeights ( m_randSeed );

        delete[] neuronsPerLayerAuto;
        delete[] neuronsPerLayerNetHidden;
        delete[] neurPerLayer;

        cout<<endl<<endl;
    }

    if ( m_isFirstEpoch == 0 )
        m_isFirstEpoch = new bool[m_nCross+1];
    for ( int i=0;i<m_nCross+1;i++ )
        m_isFirstEpoch[i] = false;
}

void NeuralNetworkRBMauto::predictAllOutputs ( REAL* rawInputs, REAL* outputs, uint nSamples, uint crossRun )
{
    // predict all samples
    for ( int i=0;i<nSamples;i++ )
        m_nn[crossRun]->predictSingleInput ( rawInputs + i*m_nFeatures, outputs + i*m_nClass*m_nDomain );
}

void NeuralNetworkRBMauto::modelUpdate ( REAL* input, REAL* target, uint nSamples, uint crossRun )
{
    m_nn[crossRun]->setTrainInputs ( input );
    m_nn[crossRun]->setTrainTargets ( target );
    m_nn[crossRun]->setNrExamplesTrain ( nSamples );

    if ( crossRun < m_nCross )
    {
        if ( m_isFirstEpoch[crossRun] == false )
        {
            m_isFirstEpoch[crossRun] = true;
            // Autoencoder: RBM pretraining + finetuning
            m_nn[crossRun]->rbmPretraining ( input, target, nSamples, m_nDomain*m_nClass, m_nrLayerAuto+1, crossRun, m_tuningEpochsRBMFinetuning, m_learnrateAutoencoderFinetuning );
        }
        else
        {
            // one gradient descent step (one epoch)
            if ( m_epoch < m_tuningEpochsOutputNet )
                //if(m_epoch > m_tuningEpochsOutputNet)
                //if(m_epoch%10 == 0)
            {
                bool* updateLayer = new bool[m_nrLayerAuto + m_nrLayerNetHidden + 3];
                for ( int i=0;i<m_nrLayerAuto;i++ )
                    updateLayer[i] = false;  // sigmoid
                updateLayer[m_nrLayerAuto] = false;  // linear
                for ( int i=0;i<m_nrLayerNetHidden;i++ )
                    updateLayer[m_nrLayerAuto+1+i] = true;  // tanh
                updateLayer[m_nrLayerAuto+m_nrLayerNetHidden+1] = true;  // tanh

                cout<<"[OutNet]";
                m_nn[crossRun]->trainOneEpoch ( updateLayer );
                delete[] updateLayer;
            }
            else
            {
                /*bool* updateLayer = new bool[m_nrLayerAuto + m_nrLayerNetHidden + 3];
                for(int i=0;i<m_nrLayerAuto;i++)
                    updateLayer[i] = true;  // sigmoid
                updateLayer[m_nrLayerAuto] = true;  // linear
                for(int i=0;i<m_nrLayerNetHidden;i++)
                    updateLayer[m_nrLayerAuto+1+i] = false;  // tanh
                updateLayer[m_nrLayerAuto+m_nrLayerNetHidden+1] = false;  // tanh

                cout<<"[In-Net]";
                m_nn[crossRun]->trainOneEpoch(updateLayer);

                delete[] updateLayer;
                */
                m_nn[crossRun]->trainOneEpoch ( 0 );
            }

            if ( crossRun == m_nCross - 1 )
                m_nn[crossRun]->printLearnrate();
        }
    }
    else
    {
        cout<<endl<<"Tune: Training of full trainset "<<endl;

        if ( m_isFirstEpoch[crossRun] == false )
        {
            m_isFirstEpoch[crossRun] = true;
            // Autoencoder: RBM pretraining + finetuning
            m_nn[crossRun]->rbmPretraining ( input, target, nSamples, m_nDomain*m_nClass, m_nrLayerAuto+1, crossRun, m_tuningEpochsRBMFinetuning, m_learnrateAutoencoderFinetuning );
        }

        // retraining with fix number of epochs
        m_nn[crossRun]->setNormalTrainStopping ( false );
        int maxEpochs = m_epochParamBest[0];
        if ( maxEpochs == 0 )
            maxEpochs = 1;  // train at least one epoch
        cout<<"Best #epochs (on cross validation): "<<maxEpochs<<endl;
        m_nn[crossRun]->setMaxEpochs ( maxEpochs );

        // train the net
        int epochs = m_nn[crossRun]->trainNN();
        cout<<endl;
    }
}

void NeuralNetworkRBMauto::saveWeights ( int cross )
{
    char buf[1024];
    sprintf ( buf,"%02d",cross );
    string name = m_datasetPath + "/" + m_tempPath + "/" + m_weightFile + "." + buf;
    if ( m_inRetraining )
        cout<<"Save:"<<name<<endl;
    int n = m_nn[cross]->getNrWeights();
    REAL* w = m_nn[cross]->getWeightPtr();

    fstream f ( name.c_str(), ios::out );
    f.write ( ( char* ) &m_nTrain, sizeof ( int ) );
    f.write ( ( char* ) &m_nFeatures, sizeof ( int ) );
    f.write ( ( char* ) &m_nClass, sizeof ( int ) );
    f.write ( ( char* ) &m_nDomain, sizeof ( int ) );
    f.write ( ( char* ) &n, sizeof ( int ) );
    f.write ( ( char* ) w, sizeof ( REAL ) *n );
    f.write ( ( char* ) &m_maxSwing, sizeof ( double ) );
    f.close();
}

void NeuralNetworkRBMauto::loadWeights ( int cross )
{
    // load weights
    char buf[1024];
    sprintf ( buf,"%02d",cross );
    string name = m_datasetPath + "/" + m_tempPath + "/" + m_weightFile + "." + buf;
    cout<<"Load:"<<name<<endl;
    fstream f ( name.c_str(), ios::in );
    if ( f.is_open() == false )
        assert ( false );
    f.read ( ( char* ) &m_nTrain, sizeof ( int ) );
    f.read ( ( char* ) &m_nFeatures, sizeof ( int ) );
    f.read ( ( char* ) &m_nClass, sizeof ( int ) );
    f.read ( ( char* ) &m_nDomain, sizeof ( int ) );

    // set up NNs (only the last one is used)
    m_nn = new NNRBM*[m_nCross+1];
    for ( int i=0;i<m_nCross+1;i++ )
        m_nn[i] = 0;
    m_nn[cross] = new NNRBM();
    m_nn[cross]->setNrTargets ( m_nClass*m_nDomain );
    m_nn[cross]->setNrInputs ( m_nFeatures );
    m_nn[cross]->setNrExamplesTrain ( 0 );
    m_nn[cross]->setNrExamplesProbe ( 0 );
    m_nn[cross]->setTrainInputs ( 0 );
    m_nn[cross]->setTrainTargets ( 0 );
    m_nn[cross]->setProbeInputs ( 0 );
    m_nn[cross]->setProbeTargets ( 0 );

    // learn parameters
    m_nn[cross]->setInitWeightFactor ( m_initWeightFactor );
    m_nn[cross]->setLearnrate ( m_learnrate );
    m_nn[cross]->setLearnrateMinimum ( m_learnrateMinimum );
    m_nn[cross]->setLearnrateSubtractionValueAfterEverySample ( m_learnrateSubtractionValueAfterEverySample );
    m_nn[cross]->setLearnrateSubtractionValueAfterEveryEpoch ( m_learnrateSubtractionValueAfterEveryEpoch );
    m_nn[cross]->setMomentum ( m_momentum );
    m_nn[cross]->setWeightDecay ( m_weightDecay );
    m_nn[cross]->setMinUpdateErrorBound ( m_minUpdateErrorBound );
    m_nn[cross]->setBatchSize ( m_batchSize );
    m_nn[cross]->setMaxEpochs ( m_maxTuninigEpochs );
    m_nn[cross]->setL1Regularization ( m_enableL1Regularization );
    m_nn[cross]->enableErrorFunctionMAE ( m_enableErrorFunctionMAE );


    // set net inner stucture
    int nrLayer = m_nrLayerAuto + m_nrLayerNetHidden;
    int* neuronsPerLayerAuto = Data::splitStringToIntegerList ( m_neuronsPerLayerAuto, ',' );
    int* neuronsPerLayerNetHidden = Data::splitStringToIntegerList ( m_neuronsPerLayerNetHidden, ',' );
    int* neurPerLayer = new int[nrLayer];
    int* layerType = new int[nrLayer+1];
    for ( int j=0;j<m_nrLayerAuto;j++ )
    {
        neurPerLayer[j] = neuronsPerLayerAuto[j];
        layerType[j] = 0;  // sig
    }
    layerType[m_nrLayerAuto] = 0;  // linear
    for ( int j=0;j<m_nrLayerNetHidden;j++ )
    {
        neurPerLayer[j+m_nrLayerAuto] = neuronsPerLayerNetHidden[j];
        layerType[j+m_nrLayerAuto+1] = 2;  // tanh
    }
    layerType[m_nrLayerNetHidden+m_nrLayerAuto+1] = 2;  // tanh

    m_nn[cross]->enableRPROP ( m_enableRPROP );
    m_nn[cross]->setNNStructure ( nrLayer+1, neurPerLayer, false, layerType );


    m_nn[cross]->setRPROPPosNeg ( m_etaPosRPROP, m_etaNegRPROP );
    m_nn[cross]->setRPROPMinMaxUpdate ( m_minUpdateRPROP, m_maxUpdateRPROP );
    m_nn[cross]->setScaleOffset ( m_scaleOutputs, m_offsetOutputs );
    m_nn[cross]->setNormalTrainStopping ( true );
    m_nn[cross]->enableRPROP ( m_enableRPROP );
    m_nn[cross]->useBLASforTraining ( m_useBLASforTraining );
    m_nn[cross]->initNNWeights ( m_randSeed );

    delete[] neuronsPerLayerAuto;
    delete[] neuronsPerLayerNetHidden;
    delete[] neurPerLayer;
    delete[] layerType;

    int n = 0;
    f.read ( ( char* ) &n, sizeof ( int ) );

    REAL* w = new REAL[n];

    f.read ( ( char* ) w, sizeof ( REAL ) *n );
    f.read ( ( char* ) &m_maxSwing, sizeof ( double ) );
    f.close();

    // init the NN weights
    m_nn[cross]->setWeights ( w );

    if ( w )
        delete[] w;
    w = 0;
}

void NeuralNetworkRBMauto::loadMetaWeights ( int cross )
{
    // nothing to do in a gradient-descent based algorithm
}

string NeuralNetworkRBMauto::templateGenerator ( int id, string preEffect, int nameID, bool blendStop )
{
    stringstream s;
    s<<"ALGORITHM=NeuralNetworkRBMauto"<<endl;
    s<<"ID="<<id<<endl;
    s<<"TRAIN_ON_FULLPREDICTOR="<<preEffect<<endl;
    s<<"DISABLE=0"<<endl;
    s<<endl;
    s<<"[int]"<<endl;
    s<<"nrLayer=3"<<endl;
    s<<"batchSize=1"<<endl;
    s<<"minTuninigEpochs=30"<<endl;
    s<<"maxTuninigEpochs=100"<<endl;
    s<<endl;
    s<<"[double]"<<endl;
    s<<"initMaxSwing=1.0"<<endl;
    s<<endl;
    s<<"offsetOutputs=0.0"<<endl;
    s<<"scaleOutputs=1.2"<<endl;
    s<<endl;
    s<<"etaPosRPROP=1.005"<<endl;
    s<<"etaNegRPROP=0.99"<<endl;
    s<<"minUpdateRPROP=1e-8"<<endl;
    s<<"maxUpdateRPROP=1e-2"<<endl;
    s<<endl;
    s<<"initWeightFactor=1.0"<<endl;
    s<<"learnrate=1e-3"<<endl;
    s<<"learnrateMinimum=1e-5"<<endl;
    s<<"learnrateSubtractionValueAfterEverySample=0.0"<<endl;
    s<<"learnrateSubtractionValueAfterEveryEpoch=0.0"<<endl;
    s<<"momentum=0.0"<<endl;
    s<<"weightDecay=0.0"<<endl;
    s<<"minUpdateErrorBound=1e-6"<<endl;
    s<<endl;
    s<<"[bool]"<<endl;
    s<<"enableErrorFunctionMAE=0"<<endl;
    s<<"enableL1Regularization=0"<<endl;
    s<<"enableClipping=1"<<endl;
    s<<"enableTuneSwing=0"<<endl;
    s<<"useBLASforTraining=1"<<endl;
    s<<"enableRPROP=0"<<endl;
    s<<endl;
    s<<"minimzeProbe="<< ( !blendStop ) <<endl;
    s<<"minimzeProbeClassificationError=0"<<endl;
    s<<"minimzeBlend="<<blendStop<<endl;
    s<<"minimzeBlendClassificationError=0"<<endl;
    s<<endl;
    s<<"[string]"<<endl;
    s<<"neuronsPerLayer=30,20,40,30,100,-1"<<endl;
    s<<"weightFile=NeuralNetworkRBMauto_"<<nameID<<"_weights.dat"<<endl;
    s<<"fullPrediction=NeuralNetworkRBMauto_"<<nameID<<".dat"<<endl;

    return s.str();
}