StandardAlgorithm.cpp

#include "StandardAlgorithm.h"

extern StreamOutput cout;

StandardAlgorithm::StandardAlgorithm()
{
    cout<<"StandardAlgorithm"<<endl;
    // init member vars
    m_blendStop = 0;
    m_maxSwing = 0;
    m_crossValidationPrediction = 0;
    m_prediction = 0;
    m_predictionBest = 0;
    m_predictionProbe = 0;
    m_singlePrediction = 0;
    m_labelPrediction = 0;
    m_wrongLabelCnt = 0;
    m_maxTuninigEpochs = 0;
    m_minTuninigEpochs = 0;
    m_enableClipping = 0;
    m_enableTuneSwing = 0;
    m_minimzeProbe = 0;
    m_minimzeProbeClassificationError = 0;
    m_minimzeBlend = 0;
    m_minimzeBlendClassificationError = 0;
    m_initMaxSwing = 0;
    m_outOfBagEstimate = 0;
    m_outOfBagEstimateCnt = 0;

}

StandardAlgorithm::~StandardAlgorithm()
{
    cout<<"descructor StandardAlgorithm"<<endl;

    if ( m_blendStop )
        delete m_blendStop;
    m_blendStop = 0;

    if ( m_prediction )
        delete[] m_prediction;
    m_prediction = 0;
    if ( m_predictionBest )
        delete[] m_predictionBest;
    m_predictionBest = 0;
    for ( int i=0;i<m_maxThreadsInCross;i++ )
    {
        if ( m_predictionProbe )
        {
            if ( m_predictionProbe[i] )
                delete[] m_predictionProbe[i];
            m_predictionProbe[i] = 0;
        }
    }
    if ( m_predictionProbe )
        delete[] m_predictionProbe;
    m_predictionProbe = 0;
    if ( m_labelPrediction )
        delete[] m_labelPrediction;
    m_labelPrediction = 0;
    if ( m_singlePrediction )
        delete[] m_singlePrediction;
    m_singlePrediction = 0;

    if ( m_crossValidationPrediction )
        delete[] m_crossValidationPrediction;
    m_crossValidationPrediction = 0;
    if ( m_wrongLabelCnt )
        delete[] m_wrongLabelCnt;
    m_wrongLabelCnt = 0;
    if(m_outOfBagEstimate)
        delete[] m_outOfBagEstimate;
    m_outOfBagEstimate = 0;
    if(m_outOfBagEstimateCnt)
        delete[] m_outOfBagEstimateCnt;
    m_outOfBagEstimateCnt = 0;
}

double StandardAlgorithm::train()
{
    cout<<"Start train StandardAlgorithm"<<endl;

    init();

    modelInit();

    double rmse = m_blendStop->calcBlending();
    cout<<endl<<"ERR Blend:"<<rmse<<endl;

    cout<<endl<<"============================ START TRAIN (param tuning) ============================="<<endl<<endl;
    cout<<"Parameters to tune:"<<endl;

    // automatically tune parameters
    m_maxSwing = m_initMaxSwing;
    for ( int i=0;i<paramEpochValues.size();i++ )
    {
        addEpochParameter ( paramEpochValues[i], paramEpochNames[i] );
        cout<<"[EPOCH] name:"<<paramEpochNames[i]<<"   initValue:"<<*paramEpochValues[i]<<endl;
    }
    for ( int i=0;i<paramDoubleValues.size();i++ )
    {
        addDoubleParameter ( paramDoubleValues[i], paramDoubleNames[i] );
        cout<<"[REAL] name:"<<paramDoubleNames[i]<<"   initValue:"<<*paramDoubleValues[i]<<endl;
    }
    for ( int i=0;i<paramIntValues.size();i++ )
    {
        addIntegerParameter ( paramIntValues[i], paramIntNames[i] );
        cout<<"[INT]  name:"<<paramIntNames[i]<<"   initValue:"<<*paramIntValues[i]<<endl;
    }
    if ( m_enableTuneSwing )
    {
        addDoubleParameter ( &m_maxSwing, "swing" );
        cout<<"[REAL] name:"<<"swing"<<"   initValue:"<<m_maxSwing<<endl;
    }

    // when in multiple optimization loop
    if ( m_loadWeightsBeforeTraining )
        loadMetaWeights ( m_nCross );

    // start the structured searcher
    cout<<"(min|max. epochs: "<<m_minTuninigEpochs<<"|"<<m_maxTuninigEpochs<<")"<<endl;
    expSearcher ( m_minTuninigEpochs, m_maxTuninigEpochs, 3, 1, 0.8, m_minimzeProbe, m_minimzeBlend );

    // remove the parameters from the searchers
    for ( int i=0;i<paramEpochValues.size();i++ )
        removeEpochParameter ( paramEpochNames[i] );
    for ( int i=0;i<paramDoubleValues.size();i++ )
        removeDoubleParameter ( paramDoubleNames[i] );
    for ( int i=0;i<paramIntValues.size();i++ )
        removeIntegerParameter ( paramIntNames[i] );
    if ( m_enableTuneSwing )
        removeDoubleParameter ( "swing" );

    paramEpochValues.clear();
    paramEpochNames.clear();
    paramDoubleValues.clear();
    paramDoubleNames.clear();
    paramIntValues.clear();
    paramIntNames.clear();


    cout<<endl<<"============================ END auto-optimize ============================="<<endl<<endl;

    // calculate all train targets with cross validation
    calculateFullPrediction();

    return expSearchGetLowestError();
}

void StandardAlgorithm::readMaps()
{
    cout<<"Read dsc maps (standard values)"<<endl;
    m_minTuninigEpochs = m_intMap["minTuninigEpochs"];
    m_maxTuninigEpochs = m_intMap["maxTuninigEpochs"];
    m_initMaxSwing = m_doubleMap["initMaxSwing"];
    m_enableClipping = m_boolMap["enableClipping"];
    m_enableTuneSwing = m_boolMap["enableTuneSwing"];
    m_minimzeProbe = m_boolMap["minimzeProbe"];
    m_minimzeProbeClassificationError = m_boolMap["minimzeProbeClassificationError"];
    m_minimzeBlend = m_boolMap["minimzeBlend"];
    m_minimzeBlendClassificationError = m_boolMap["minimzeBlendClassificationError"];
    m_weightFile = m_stringMap["weightFile"];
    m_fullPrediction = m_stringMap["fullPrediction"];
}

void StandardAlgorithm::init()
{
    cout<<"Init standard algorithm"<<endl;

    // read standard and specific values
    readMaps();
    readSpecificMaps();

    if ( m_blendStop == 0 )
    {
        // init blendStop data
        m_blendStop = new BlendStopping ( this, "tune" );
        m_blendStop->setRegularization ( m_blendingRegularization );

        m_wrongLabelCnt = new int[m_nDomain];
        m_singlePrediction = new REAL[m_nClass * m_nDomain];

        if(m_validationType == "ValidationSet")
        {
            m_prediction = new REAL[m_validSize * m_nClass * m_nDomain];
            m_predictionBest = new REAL[m_validSize * m_nClass * m_nDomain];
            m_labelPrediction = new int[m_validSize * m_nDomain];
            return;
        }
        
        // prediction on trainset for all classes
        m_prediction = new REAL[m_nTrain * m_nClass * m_nDomain];
        m_predictionBest = new REAL[m_nTrain * m_nClass * m_nDomain];
        m_predictionProbe = new REAL*[m_maxThreadsInCross];
        for ( int i=0;i<m_maxThreadsInCross;i++ )
            m_predictionProbe[i] = new REAL[m_nTrain * m_nClass * m_nDomain];
        m_labelPrediction = new int[m_nTrain * m_nDomain];
        
        // cross validation training
        m_crossValidationPrediction = new REAL[m_nTrain*m_nClass*m_nDomain];

        if(m_validationType == "Bagging")
        {
            m_outOfBagEstimate = new REAL[m_nTrain * m_nClass * m_nDomain];
            m_outOfBagEstimateCnt = new int[m_nTrain];
        }
    }
}

double StandardAlgorithm::calcRMSEonProbe()
{
    double rmse = 0.0, mae = 0.0;
    int nThreads = m_maxThreadsInCross;  // get #available threads
    for ( int d=0;d<m_nDomain;d++ )
        m_wrongLabelCnt[d] = 0;

    if(m_validationType == "ValidationSet")
    {
        // train the model on the train set
        modelUpdate ( m_trainOrig, m_trainTargetOrig, m_nTrain, 0 );
        
        // predict the validation set
        REAL* effect = new REAL[m_validSize * m_nClass * m_nDomain];
        for(uint i=0;i<m_validSize * m_nClass * m_nDomain;i++)
            effect[i] = 0.0;
        predictMultipleOutputs ( m_valid, effect, m_prediction, m_labelPrediction, m_validSize, 0 );
        delete[] effect;
        
        // calculate the error on the validation set
        for ( int i=0;i<m_validSize;i++)
        {
            // copy to blending vector and to internal prediction
            for ( int j=0;j<m_nClass*m_nDomain;j++ )
            {
                REAL prediction = m_prediction[i*m_nClass*m_nDomain + j];
                m_blendStop->m_newPrediction[j][i] = prediction;
                m_prediction[i*m_nClass*m_nDomain + j] = prediction;
                
                rmse += ( prediction - m_validTarget[i*m_nClass*m_nDomain + j] ) * ( prediction - m_validTarget[i*m_nClass*m_nDomain + j] );
                mae += fabs ( prediction - m_validTarget[i*m_nClass*m_nDomain + j] );
            }
            
            // count wrong labeled examples
            if ( Framework::getDatasetType() )
            {
                for ( int d=0;d<m_nDomain;d++ )
                    if ( m_labelPrediction[d+i*m_nDomain] != m_validLabel[d+i*m_nDomain] )
                        m_wrongLabelCnt[d]++;
            }
        }
        
        // print the classification error rate
        double classificationError = 1.0;
        if ( Framework::getDatasetType() )
        {
            int nWrong = 0;
            for ( int d=0;d<m_nDomain;d++ )
            {
                nWrong += m_wrongLabelCnt[d];
                //if(m_nDomain > 1)
                //    cout<<"["<<(double)m_wrongLabelCnt[d]/(double)m_validSize<<"] ";
            }
            classificationError = ( double ) nWrong/ ( ( double ) m_validSize*m_nDomain );
            cout<<" [classErr:"<<100.0*classificationError<<"%] ";
        }
        if ( m_minimzeProbeClassificationError )
            return classificationError;

        rmse = sqrt ( rmse/ ( ( double ) m_validSize * ( double ) m_nClass * ( double ) m_nDomain ) );
        mae = mae/ ( ( double ) m_validSize * ( double ) m_nClass * ( double ) m_nDomain );

        if ( m_errorFunction=="MAE" )
            return mae;
        else if ( m_errorFunction=="AUC" && Framework::getDatasetType() )
        {
            cout<<"[rmse:"<<rmse<<"]"<<flush;

            // calc area under curve
            REAL* tmp = new REAL[m_validSize*m_nClass*m_nDomain];
            for ( int i=0;i<m_validSize;i++ )
                for ( int j=0;j<m_nClass*m_nDomain;j++ )
                    tmp[i + j*m_validSize] = m_prediction[j + i*m_nClass*m_nDomain];
            REAL auc = getAUC ( tmp, m_validLabel, m_nClass, m_nDomain, m_validSize );
            delete[] tmp;
            return auc;
        }

        return rmse;
    }
    
    if(m_validationType == "Bagging")
    {
        for(int i=0;i<m_nTrain * m_nClass * m_nDomain;i++)
            m_outOfBagEstimate[i] = 0.0;
        for(int i=0;i<m_nTrain;i++)
            m_outOfBagEstimateCnt[i] = 0;
        for(int i=0;i<m_nTrain;i++)
            for(int j=0;j<m_nClass * m_nDomain;j++)
                m_prediction[i*m_nClass*m_nDomain+j] = m_targetMean[j];  // some of the out-of-bag estimates are not predicted
    }
    
    for ( int i=0;i<m_nCross;i+=nThreads ) // all cross validation sets
    {
        // predict the probeset
        int* nSamples = new int[nThreads];
        int** labels = new int*[nThreads];
        for ( int j=0;j<nThreads;j++ )
        {
            nSamples[j] = m_probeSize[i+j];
            labels[j] = new int[nSamples[j]*m_nDomain];
        }

        if ( nThreads > 1 )
        {
            // parallel training of the cross-validation sets with OPENMP
#pragma omp parallel for
            for ( int t=0;t<nThreads;t++ )
            {
                cout<<"."<<flush;
                if ( m_enableSaveMemory )
                    fillNCrossValidationSet ( i+t );
                modelUpdate ( m_train[i+t], m_trainTargetResidual[i+t], m_trainSize[i+t], i+t );
                predictMultipleOutputs ( m_probe[i+t], m_probeTargetEffect[i+t], m_predictionProbe[t], labels[t], nSamples[t], i+t );
                if ( m_enableSaveMemory )
                    freeNCrossValidationSet ( i+t );
            }
        }
        else
        {
            cout<<"."<<flush;
            if ( m_enableSaveMemory )
                fillNCrossValidationSet ( i );
            modelUpdate ( m_train[i], m_trainTargetResidual[i], m_trainSize[i], i );
            predictMultipleOutputs ( m_probe[i], m_probeTargetEffect[i], m_predictionProbe[0], labels[0], nSamples[0], i );
            if ( m_enableSaveMemory )
                freeNCrossValidationSet ( i );
        }

        // merge the probe predictions
        if(m_validationType == "Bagging")
        {
            for ( int thread=0;thread<nThreads;thread++ )
            {
                for ( int j=0;j<nSamples[thread];j++ )
                {
                    int idx = m_probeIndex[i+thread][j];
                    for(int k=0;k<m_nClass * m_nDomain;k++)
                    {
                        REAL prediction = m_predictionProbe[thread][j*m_nClass*m_nDomain + k];
                        m_outOfBagEstimate[idx*m_nClass*m_nDomain + k] += prediction;
                    }
                    m_outOfBagEstimateCnt[idx]++;
                }
            }
        }
        else
        {
            for ( int thread=0;thread<nThreads;thread++ )
            {
                for ( int j=0;j<nSamples[thread];j++ ) // for all samples in this set
                {
                    int idx = m_probeIndex[i+thread][j];
    
                    // copy to blending vector and to internal prediction
                    for ( int k=0;k<m_nClass*m_nDomain;k++ )
                    {
                        REAL prediction = m_predictionProbe[thread][j*m_nClass*m_nDomain + k];
                        m_blendStop->m_newPrediction[k][idx] = prediction;
                        m_prediction[idx*m_nClass*m_nDomain + k] = prediction;
                        rmse += ( prediction - m_probeTarget[i+thread][m_nClass*m_nDomain*j + k] ) * ( prediction - m_probeTarget[i+thread][m_nClass*m_nDomain*j + k] );
                        mae += fabs ( prediction - m_probeTarget[i+thread][m_nClass*m_nDomain*j + k] );
                    }
    
                    // count wrong labeled examples
                    if ( Framework::getDatasetType() )
                    {
                        for ( int d=0;d<m_nDomain;d++ )
                            if ( labels[thread][d+j*m_nDomain] != m_probeLabel[i+thread][d+j*m_nDomain] )
                                m_wrongLabelCnt[d]++;
                    }
                }
            }
        }
        
        // free memory
        for ( int j=0;j<nThreads;j++ )
        {
            if ( labels[j] )
                delete[] labels[j];
            labels[j] = 0;
        }
        if ( nSamples )
            delete[] nSamples;
        nSamples = 0;
        if ( labels )
            delete[] labels;
        labels = 0;

    }

    if(m_validationType == "Bagging")
    {
        for(int i=0;i<m_nTrain;i++)
        {
            int c = m_outOfBagEstimateCnt[i];
            for(int j=0;j<m_nClass*m_nDomain;j++)
                m_prediction[i*m_nClass*m_nDomain+j] = (c==0 ? m_targetMean[j] : (m_outOfBagEstimate[i*m_nClass*m_nDomain+j] / (REAL)c));
            
            for(int j=0;j<m_nClass*m_nDomain;j++)
            {
                REAL prediction = m_prediction[i*m_nClass*m_nDomain+j];
                m_blendStop->m_newPrediction[j][i] = prediction;
                rmse += ( prediction - m_trainTargetOrig[m_nClass*m_nDomain*i + j] ) * ( prediction - m_trainTargetOrig[m_nClass*m_nDomain*i + j] );
                mae += fabs ( prediction - m_trainTargetOrig[m_nClass*m_nDomain*i + j] );
            }
            
             // count wrong labeled examples
            if ( Framework::getDatasetType() )
            {
                for(int j=0;j<m_nDomain;j++)
                {
                    int indBest = -1;
                    REAL max = -1e10;
                    for(int k=0;k<m_nClass;k++)
                    {
                        if(max < m_prediction[i*m_nClass + j*m_nClass + k])
                        {
                            max = m_prediction[i*m_nClass + j*m_nClass + k];
                            indBest = k;
                        }
                    }
                    if(indBest != m_trainLabelOrig[i+j*m_nClass])
                        m_wrongLabelCnt[j]++;
                }
            }
        }
    }
    
    // print the classification error rate
    double classificationError = 1.0;
    if ( Framework::getDatasetType() )
    {
        int nWrong = 0;
        for ( int d=0;d<m_nDomain;d++ )
        {
            nWrong += m_wrongLabelCnt[d];
            //if(m_nDomain > 1)
            //    cout<<"["<<(double)m_wrongLabelCnt[d]/(double)m_nTrain<<"] ";
        }
        classificationError = ( double ) nWrong/ ( ( double ) m_nTrain*m_nDomain );
        cout<<" [classErr:"<<100.0*classificationError<<"%] ";
    }
    if ( m_minimzeProbeClassificationError )
        return classificationError;

    rmse = sqrt ( rmse/ ( ( double ) m_nTrain * ( double ) m_nClass * ( double ) m_nDomain ) );
    mae = mae/ ( ( double ) m_nTrain * ( double ) m_nClass * ( double ) m_nDomain );

    if ( m_errorFunction=="MAE" )
        return mae;
    else if ( m_errorFunction=="AUC" && Framework::getDatasetType() )
    {
        cout<<"[rmse:"<<rmse<<"]"<<flush;

        // calc area under curve
        REAL* tmp = new REAL[m_nTrain*m_nClass*m_nDomain];
        for ( int i=0;i<m_nTrain;i++ )
            for ( int j=0;j<m_nClass*m_nDomain;j++ )
                tmp[i + j*m_nTrain] = m_prediction[j + i*m_nClass*m_nDomain];
        REAL auc = getAUC ( tmp, m_trainLabelOrig, m_nClass, m_nDomain, m_nTrain );
        delete[] tmp;
        return auc;
    }

    return rmse;
}

void StandardAlgorithm::saveBestPrediction()
{
    cout<<"[saveBest]";
    memcpy ( m_predictionBest, m_prediction, sizeof ( REAL ) * (m_validationType == "ValidationSet" ? m_validSize : m_nTrain) * m_nClass * m_nDomain );
    m_blendStop->saveTmpBestWeights();
    if(m_validationType == "ValidationSet")
        saveWeights(0);
    else
    {
        if ( m_validationType == "CrossFoldMean" || m_validationType == "Bagging" )
            for ( int i=0;i<m_nCross;i++ )
                saveWeights ( i );
    }
}

double StandardAlgorithm::calcRMSEonBlend()
{
    double rmse = calcRMSEonProbe();
    cout<<" [probe:"<<rmse<<"] ";
    double rmseBlend = m_blendStop->calcBlending();
    if ( m_minimzeBlendClassificationError )
        return m_blendStop->getClassificationError();
    return rmseBlend;
}

void StandardAlgorithm::setPredictionMode ( int cross )
{
    cout<<"Set algorithm in prediction mode"<<endl;
    readMaps();
    readSpecificMaps();
    loadWeights ( cross );
}

void StandardAlgorithm::predictMultipleOutputs ( REAL* rawInput, REAL* effect, REAL* output, int* labels, int nSamples, int crossRun )
{
    // model prediction
    predictAllOutputs ( rawInput, output, nSamples, crossRun );

    if ( m_enableTuneSwing ) // clip the output (clip swing)
    {
        IPPS_THRESHOLD ( output, output, nSamples*m_nClass*m_nDomain, -m_maxSwing, ippCmpLess );  // clip negative
        IPPS_THRESHOLD ( output, output, nSamples*m_nClass*m_nDomain, +m_maxSwing, ippCmpGreater );  // clip positive
    }

    // add the output from the preprocessor (=effect)
    V_ADD ( nSamples*m_nClass*m_nDomain, output, effect, output );

    // calc output labels (for classification dataset)
    if ( Framework::getDatasetType() )
    {
        // in all domains
        for ( int d=0;d<m_nDomain;d++ )
        {
            // calc output labels
            for ( int i=0;i<nSamples;i++ )
            {
                // find max. output value
                int indMax = -1;
                REAL max = -1e10;
                for ( int j=0;j<m_nClass;j++ )
                {
                    if ( max < output[d*m_nClass + i*m_nDomain*m_nClass + j] )
                    {
                        max = output[d*m_nClass + i*m_nDomain*m_nClass + j];
                        indMax = j;
                    }
                }
                labels[d+i*m_nDomain] = indMax;
            }
        }
    }

    // clip final outputs
    if ( m_enableClipping )
    {
        IPPS_THRESHOLD ( output, output, nSamples*m_nClass*m_nDomain, m_negativeTarget, ippCmpLess );  // clip negative
        IPPS_THRESHOLD ( output, output, nSamples*m_nClass*m_nDomain, m_positiveTarget, ippCmpGreater );  // clip positive
    }

    // add small noise
    if ( m_addOutputNoise > 0.0 )
        for ( int i=0;i<nSamples*m_nClass*m_nDomain;i++ )
            output[i] += NumericalTools::getNormRandomNumber ( 0.0, m_addOutputNoise );

}

void StandardAlgorithm::writeFullPrediction(int nSamples)
{
    // calc train RMSE
    double rmse = 0.0, err;
    for ( int i=0;i<nSamples;i++ )
    {
        for ( int j=0;j<m_nClass*m_nDomain;j++ )
        {
            err = m_prediction[j+i*m_nClass*m_nDomain] - m_trainTargetOrig[j+i*m_nClass*m_nDomain];
            rmse += err*err;
        }
    }

    // write file
    string name = m_datasetPath + "/" + m_fullPredPath + "/" + m_fullPrediction;
    cout<<"Write full prediction: "<<name<<" (RMSE:"<<sqrt ( rmse/ ( double ) ( nSamples*m_nClass*m_nDomain ) ) <<")";
    fstream f;
    f.open ( name.c_str(),ios::out );
    f.write ( ( char* ) m_prediction, sizeof ( REAL ) *nSamples*m_nClass*m_nDomain );
    f.close();
    cout<<endl;
}

void StandardAlgorithm::calculateFullPrediction()
{
    double rmse = 0.0;
    cout<<endl<<"Calculate FullPrediction (write the prediction of the trainingset with cross validation)"<<endl<<endl;

    // re-calculate the blending weights (necessary for minimize only probe)
    if ( m_minimzeProbe )
    {
        double rmseBlend = m_blendStop->calcBlending();
        m_blendStop->saveTmpBestWeights();
        cout<<"rmseBlend:"<<rmseBlend<<endl;
    }

    // save linear blending weights
    m_blendStop->saveBlendingWeights ( m_datasetPath + "/" + m_tempPath, true );
    cout<<endl;

    memcpy ( m_prediction, m_predictionBest, sizeof ( REAL ) * (m_validationType == "ValidationSet"?m_validSize:m_nTrain) * m_nClass * m_nDomain );
    writeFullPrediction(m_validationType == "ValidationSet"?m_validSize:m_nTrain);

    m_inRetraining = true;

    if ( m_validationType == "Retraining" )
    {
        cout<<"Validation type: Retraining"<<endl;
        cout<<"Update model on whole training set"<<endl<<endl;
        time_t retrainTime = time ( 0 );

        // retrain the model with whole trainingset (disable cross validation)
        if ( m_enableSaveMemory )
            fillNCrossValidationSet ( m_nCross );

        // tmp variables, used for bagging
        REAL* trainOrig = 0;
        REAL* targetOrig = 0;
        REAL* targetEffectOrig = 0;
        REAL* targetResidualOrig = 0;
        int* labelOrig = 0;

        if ( m_enableBagging )
        {
            cout<<"Save orig data, create boostrap sample for retraining"<<endl;
            trainOrig = m_train[m_nCross];
            targetOrig = m_trainTarget[m_nCross];
            targetEffectOrig = m_trainTargetEffect[m_nCross];
            targetResidualOrig = m_trainTargetResidual[m_nCross];
            labelOrig = m_trainLabel[m_nCross];
            doBootstrapSampling ( 0, m_train[m_nCross], m_trainTarget[m_nCross], m_trainTargetEffect[m_nCross], m_trainTargetResidual[m_nCross], m_trainLabel[m_nCross] ); // bootstrap sample
            //doBootstrapSampling(0, m_train[m_nCross], m_trainTarget[m_nCross], m_trainTargetEffect[m_nCross], m_trainTargetResidual[m_nCross], m_trainLabel[m_nCross], m_nTrain * 0.8);
        }

        modelUpdate ( m_train[m_nCross], m_trainTargetResidual[m_nCross], m_nTrain, m_nCross );
        saveWeights ( m_nCross );

        // calc retrain rmse
        cout<<"Calculate retrain RMSE (on trainset)"<<endl;
        rmse = 0.0;
        memset ( m_prediction, 0, sizeof ( REAL ) *m_nTrain*m_nClass*m_nDomain );
        predictMultipleOutputs ( m_train[m_nCross], m_trainTargetEffect[m_nCross], m_prediction, m_labelPrediction, m_nTrain, m_nCross );
        for ( int i=0;i<m_nTrain*m_nClass*m_nDomain;i++ )
            rmse += ( m_prediction[i] - m_trainTarget[m_nCross][i] ) * ( m_prediction[i] - m_trainTarget[m_nCross][i] );
        rmse = sqrt ( rmse/ ( double ) ( m_nTrain * m_nClass * m_nDomain ) );
        cout<<"Train of this algorithm (RMSE after retraining): "<<rmse<<endl;

        if ( m_enableBagging )
        {
            cout<<"Restore orig data"<<endl;
            if ( m_train[m_nCross] )
                delete[] m_train[m_nCross];
            if ( m_trainTarget[m_nCross] )
                delete[] m_trainTarget[m_nCross];
            if ( m_trainTargetEffect[m_nCross] )
                delete[] m_trainTargetEffect[m_nCross];
            if ( m_trainTargetResidual[m_nCross] )
                delete[] m_trainTargetResidual[m_nCross];
            if ( m_trainLabel[m_nCross] )
                delete[] m_trainLabel[m_nCross];
            m_train[m_nCross] = trainOrig;
            m_trainTarget[m_nCross] = targetOrig;
            m_trainTargetEffect[m_nCross] = targetEffectOrig;
            m_trainTargetResidual[m_nCross] = targetResidualOrig;
            m_trainLabel[m_nCross] = labelOrig;
        }

        if ( m_enableSaveMemory )
            freeNCrossValidationSet ( m_nCross );

        cout<<"Total retrain time:"<<time ( 0 )-retrainTime<<"[s]"<<endl;
    }
    /*else if(m_validationType == "CrossFoldMean" || m_validationType == "Bagging")
    {
        cout<<"Validation type: "<<m_validationType<<endl;
        for(int i=0;i<m_nCross;i++)
            saveWeights(i);
    }
    else
        assert(false);
    */

    // print summary
    cout<<endl<<"==========================================================================="<<endl;
    BlendStopping bb ( this, m_fullPrediction );
    bb.setRegularization ( m_blendingRegularization );
    if ( m_datasetName=="NETFLIX" && Framework::getAdditionalStartupParameter() >= 0 )
    {
        cout<<"Dataset:NETFLIX, slot:"<<Framework::getAdditionalStartupParameter() <<" ";
        char buf[512];
        sprintf ( buf,"p%d",Framework::getAdditionalStartupParameter() );
        string pName = string ( NETFLIX_SLOTDATA_ROOT_DIR ) + buf + "/trainPrediction.data";
        cout<<"pName:"<<pName<<endl;
        rmse = bb.calcBlending ( ( char* ) pName.c_str() );
    }
    else
        rmse = bb.calcBlending ( ( char* ) ( m_datasetPath + "/" + m_tempPath + "/trainPrediction.data" ).c_str() );
    bb.saveBlendingWeights ( m_datasetPath + "/" + m_tempPath );
    cout<<endl<<"BLEND RMSE OF ACTUAL FULLPREDICTION PATH:"<<rmse<<endl;
    cout<<"==========================================================================="<<endl<<endl;

}