Data.cpp

#include "Data.h"

extern StreamOutput cout;

Data::Data()
{
    cout<<"Constructor Data"<<endl;

    // init member vars
    m_algorithmID = 0;
    m_randSeed = 0;
    m_nMixDataset = 0;
    m_nMixTrainList = 0;
    m_nCross = 0;
    m_validationType = "Retraining";
    m_maxThreadsInCross = 0;
    m_enableGlobalMeanStdEstimate = 0;
    m_positiveTarget = 0;
    m_negativeTarget = 0;
    m_blendingRegularization = 0;
    m_enableGlobalBlendingWeights = 0;
    m_blendingEnableCrossValidation = 0;
    m_enablePostNNBlending = 0;
    m_enableCascadeLearning = 0;
    m_nCascadeInputs = 0;
    m_cascadeInputs = 0;
    m_nFeatures = 0;
    m_nClass = 0;
    m_nDomain = 0;
    m_mixDatasetIndices = 0;
    m_mixList = 0;
    m_crossIndex = 0;
    m_nTrain = 0;
    m_trainOrig = 0;
    m_trainTargetOrig = 0;
    m_trainTargetOrigEffect = 0;
    m_trainTargetOrigResidual = 0;
    m_trainLabelOrig = 0;
    m_trainBaggingIndex = 0;
    m_nTest = 0;
    m_testOrig = 0;
    m_testTargetOrig = 0;
    m_testLabelOrig = 0;
    m_slotBoundaries = 0;
    m_trainSize = 0;
    m_train = 0;
    m_trainTarget = 0;
    m_trainTargetEffect = 0;
    m_trainTargetResidual = 0;
    m_trainLabel = 0;
    m_probeSize = 0;
    m_probe = 0;
    m_probeTarget = 0;
    m_probeTargetEffect = 0;
    m_probeTargetResidual = 0;
    m_probeLabel = 0;
    m_probeIndex = 0;
    m_validSize = 0;
    m_valid = 0;
    m_validTarget = 0;
    m_validLabel = 0;
    m_mean = 0;
    m_std = 0;
    m_standardDeviationMin = 0;
    m_targetMean = 0;
    m_enableSaveMemory = 0;
    m_support = 0;
    m_enablePostBlendClipping = 0;
    m_addOutputNoise = 0;
    m_enableFeatureSelection = 0;
    m_featureSelectionWriteBinaryDataset = 0;
    m_enableBagging = 0;
    m_randomSeedBagging = 0;
    m_enableStaticNormalization = 0;
    m_staticMeanNormalization = 0.0;
    m_staticStdNormalization = 1.0;
    m_enableProbablisticNormalization = 0;
    m_dimensionalityReduction = "";
    m_subsampleTrainSet = 1.0;
    m_subsampleFeatures = 1.0;
    m_disableTraining = false;
    m_globalTrainingLoops = 1;
    m_addConstantInput = 0;
    m_loadWeightsBeforeTraining = false;
}

Data::~Data()
{
    cout<<"destructor Data"<<endl;

}

void Data::deleteMemory()
{
    cout<<"Delete internal memory"<<endl;

    // memory from dataset
    if ( m_trainOrig )
        delete[] m_trainOrig;
    m_trainOrig = 0;
    if ( m_trainTargetOrig )
        delete[] m_trainTargetOrig;
    m_trainTargetOrig = 0;
    if ( m_trainLabelOrig )
        delete[] m_trainLabelOrig;
    m_trainLabelOrig = 0;
    if ( m_testOrig )
        delete[] m_testOrig;
    m_testOrig = 0;
    if ( m_testTargetOrig )
        delete[] m_testTargetOrig;
    m_testTargetOrig = 0;
    if ( m_testLabelOrig )
        delete[] m_testLabelOrig;
    m_testLabelOrig = 0;

    // memory from cross validation
    if ( m_mean )
        delete[] m_mean;
    m_mean = 0;
    if ( m_std )
        delete[] m_std;
    m_std = 0;
    if ( m_trainTargetOrigEffect )
        delete[] m_trainTargetOrigEffect;
    m_trainTargetOrigEffect = 0;
    if ( m_trainTargetOrigResidual )
        delete[] m_trainTargetOrigResidual;
    m_trainTargetOrigResidual = 0;

    for ( int i=0;i<m_nCross+1;i++ )
    {
        if ( m_train )
        {
            if ( m_train[i] )
                delete[] m_train[i];
            m_train[i] = 0;
        }
        if ( m_trainTarget )
        {
            if ( m_trainTarget[i] )
                delete[] m_trainTarget[i];
            m_trainTarget[i] = 0;
        }
        if ( m_trainTargetEffect )
        {
            if ( m_trainTargetEffect[i] )
                delete[] m_trainTargetEffect[i];
            m_trainTargetEffect[i] = 0;
        }
        if ( m_trainTargetResidual )
        {
            if ( m_trainTargetResidual[i] )
                delete[] m_trainTargetResidual[i];
            m_trainTargetResidual[i] = 0;
        }
        if ( m_trainLabel )
        {
            if ( m_trainLabel[i] )
                delete[] m_trainLabel[i];
            m_trainLabel[i] = 0;
        }
        if ( m_validationType == "Bagging" )
        {
            if( m_trainBaggingIndex )
            {
                if ( m_trainBaggingIndex[i] )
                    delete[] m_trainBaggingIndex[i];
                m_trainBaggingIndex[i] = 0;
            }
        }
        if ( m_probe )
        {
            if ( m_probe[i] )
                delete[] m_probe[i];
            m_probe[i] = 0;
        }
        if ( m_probeTarget )
        {
            if ( m_probeTarget[i] )
                delete[] m_probeTarget[i];
            m_probeTarget[i] = 0;
        }
        if ( m_probeTargetEffect )
        {
            if ( m_probeTargetEffect[i] )
                delete[] m_probeTargetEffect[i];
            m_probeTargetEffect[i] = 0;
        }
        if ( m_probeTargetResidual )
        {
            if ( m_probeTargetResidual[i] )
                delete[] m_probeTargetResidual[i];
            m_probeTargetResidual[i] = 0;
        }
        if ( m_probeLabel )
        {
            if ( m_probeLabel[i] )
                delete[] m_probeLabel[i];
            m_probeLabel[i] = 0;
        }
        if ( m_probeIndex )
        {
            if ( m_probeIndex[i] )
                delete[] m_probeIndex[i];
            m_probeIndex[i] = 0;
        }
    }
    if ( m_train )
        delete[] m_train;
    m_train = 0;
    if ( m_trainTarget )
        delete[] m_trainTarget;
    m_trainTarget = 0;
    if ( m_trainTargetEffect )
        delete[] m_trainTargetEffect;
    m_trainTargetEffect = 0;
    if ( m_trainTargetResidual )
        delete[] m_trainTargetResidual;
    m_trainTargetResidual = 0;
    if ( m_trainLabel )
        delete[] m_trainLabel;
    m_trainLabel = 0;
    if(m_validationType == "Bagging")
    {
        if(m_trainBaggingIndex)
            delete[] m_trainBaggingIndex;
        m_trainBaggingIndex = 0;
    }
    if ( m_probe )
        delete[] m_probe;
    m_probe = 0;
    if ( m_probeTarget )
        delete[] m_probeTarget;
    m_probeTarget = 0;
    if ( m_probeTargetEffect )
        delete[] m_probeTargetEffect;
    m_probeTargetEffect = 0;
    if ( m_probeTargetResidual )
        delete[] m_probeTargetResidual;
    m_probeTargetResidual = 0;
    if ( m_probeLabel )
        delete[] m_probeLabel;
    m_probeLabel = 0;
    if ( m_probeIndex )
        delete[] m_probeIndex;
    m_probeIndex = 0;

    if ( m_trainSize )
        delete[] m_trainSize;
    m_trainSize = 0;
    if ( m_probeSize )
        delete[] m_probeSize;
    m_probeSize = 0;

    if ( m_mixDatasetIndices )
        delete[] m_mixDatasetIndices;
    m_mixDatasetIndices = 0;
    if ( m_mixList )
        delete[] m_mixList;
    m_mixList = 0;
    if ( m_slotBoundaries )
        delete[] m_slotBoundaries;
    m_slotBoundaries = 0;
    if ( m_crossIndex )
        delete[] m_crossIndex;
    m_crossIndex = 0;

    if ( m_cascadeInputs )
        delete[] m_cascadeInputs;
    m_cascadeInputs = 0;
    
    if ( m_targetMean )
        delete[] m_targetMean;
    m_targetMean = 0;

}

void Data::readDataset ( string name )
{
    // read MNIST
    if ( name == "MNIST" )
    {
        DatasetReader r;
        // call by reference, memory is allcated in the DatasetReader
        r.readMNIST ( m_datasetPath+"/"+m_dataPath, m_trainOrig, m_trainTargetOrig, m_trainLabelOrig, m_testOrig, m_testTargetOrig, m_testLabelOrig, m_nTrain, m_nTest, m_nClass, m_nDomain, m_nFeatures, m_positiveTarget, m_negativeTarget );
    }
    else if ( name == "NETFLIX" ) // read Netflix
    {
        DatasetReader r;
        // call by reference, memory is allcated in the DatasetReader
        r.readNETFLIX ( m_datasetPath+"/"+m_dataPath, m_trainOrig, m_trainTargetOrig, m_trainLabelOrig, m_testOrig, m_testTargetOrig, m_testLabelOrig, m_nTrain, m_nTest, m_nClass, m_nDomain, m_nFeatures, m_positiveTarget, m_negativeTarget );
    }
    else if ( name == "AusDM2009" ) // read AusDM2009
    {
        DatasetReader r;
        // call by reference, memory is allcated in the DatasetReader
        r.readAusDM2009 ( m_datasetPath+"/"+m_dataPath, m_trainOrig, m_trainTargetOrig, m_trainLabelOrig, m_testOrig, m_testTargetOrig, m_testLabelOrig, m_nTrain, m_nTest, m_nClass, m_nDomain, m_nFeatures, m_positiveTarget, m_negativeTarget );
    }
    else if ( name == "KDDCup09Large" ) // read large KDDCup09large dataset
    {
        DatasetReader r;
        // call by reference, memory is allcated in the DatasetReader
        r.readKDDCup09Large ( m_datasetPath+"/"+m_dataPath, m_trainOrig, m_trainTargetOrig, m_trainLabelOrig, m_testOrig, m_testTargetOrig, m_testLabelOrig, m_nTrain, m_nTest, m_nClass, m_nDomain, m_nFeatures, m_positiveTarget, m_negativeTarget );
    }
    else if ( name == "KDDCup09Small" ) // read large KDDCup09small dataset
    {
        DatasetReader r;
        // call by reference, memory is allcated in the DatasetReader
        r.readKDDCup09Small ( m_datasetPath+"/"+m_dataPath, m_trainOrig, m_trainTargetOrig, m_trainLabelOrig, m_testOrig, m_testTargetOrig, m_testLabelOrig, m_nTrain, m_nTest, m_nClass, m_nDomain, m_nFeatures, m_positiveTarget, m_negativeTarget );
    }
    else if ( name == "BINARY" ) // read binary format dataset
    {
        DatasetReader r;
        // call by reference, memory is allcated in the DatasetReader
        r.readBINARY ( m_datasetPath+"/"+m_dataPath, m_trainOrig, m_trainTargetOrig, m_trainLabelOrig, m_testOrig, m_testTargetOrig, m_testLabelOrig, m_nTrain, m_nTest, m_nClass, m_nDomain, m_nFeatures, m_positiveTarget, m_negativeTarget );
    }
    else if ( name == "CSV" ) // read csv format dataset
    {
        DatasetReader r;
        // call by reference, memory is allcated in the DatasetReader
        r.readCSV ( m_datasetPath+"/"+m_dataPath, m_trainOrig, m_trainTargetOrig, m_trainLabelOrig, m_testOrig, m_testTargetOrig, m_testLabelOrig, m_nTrain, m_nTest, m_nClass, m_nDomain, m_nFeatures, m_positiveTarget, m_negativeTarget );
    }
    else if ( name == "ARFF" ) // read arff format dataset
    {
        DatasetReader r;
        // call by reference, memory is allcated in the DatasetReader
        r.readARFF ( m_datasetPath+"/"+m_dataPath, m_trainOrig, m_trainTargetOrig, m_trainLabelOrig, m_testOrig, m_testTargetOrig, m_testLabelOrig, m_nTrain, m_nTest, m_nClass, m_nDomain, m_nFeatures, m_positiveTarget, m_negativeTarget );
    }
    else if ( name == "PRUDSYS_DMC2009" ) // read PRUDSYS_DMC2009 dataset
    {
        DatasetReader r;
        // call by reference, memory is allcated in the DatasetReader
        r.readPRUDSYS ( m_datasetPath+"/"+m_dataPath, m_trainOrig, m_trainTargetOrig, m_trainLabelOrig, m_testOrig, m_testTargetOrig, m_testLabelOrig, m_nTrain, m_nTest, m_nClass, m_nDomain, m_nFeatures, m_positiveTarget, m_negativeTarget );
    }
    else if ( name == "ADULT" ) // read adult dataset
    {
        DatasetReader r;
        // call by reference, memory is allcated in the DatasetReader
        r.readADULT ( m_datasetPath+"/"+m_dataPath, m_trainOrig, m_trainTargetOrig, m_trainLabelOrig, m_testOrig, m_testTargetOrig, m_testLabelOrig, m_nTrain, m_nTest, m_nClass, m_nDomain, m_nFeatures, m_positiveTarget, m_negativeTarget );
    }
    else if ( name == "AUSTRALIAN" ) // read australian dataset
    {
        DatasetReader r;
        // call by reference, memory is allcated in the DatasetReader
        r.readAUSTRALIAN ( m_datasetPath+"/"+m_dataPath, m_trainOrig, m_trainTargetOrig, m_trainLabelOrig, m_testOrig, m_testTargetOrig, m_testLabelOrig, m_nTrain, m_nTest, m_nClass, m_nDomain, m_nFeatures, m_positiveTarget, m_negativeTarget );
    }
    else if ( name == "BALANCE" ) // read balance dataset
    {
        DatasetReader r;
        // call by reference, memory is allcated in the DatasetReader
        r.readBALANCE ( m_datasetPath+"/"+m_dataPath, m_trainOrig, m_trainTargetOrig, m_trainLabelOrig, m_testOrig, m_testTargetOrig, m_testLabelOrig, m_nTrain, m_nTest, m_nClass, m_nDomain, m_nFeatures, m_positiveTarget, m_negativeTarget );
    }
    else if ( name == "CYLINDER-BANDS" ) // read cylinder-bands dataset
    {
        DatasetReader r;
        // call by reference, memory is allcated in the DatasetReader
        r.readCYLINDERBANDS ( m_datasetPath+"/"+m_dataPath, m_trainOrig, m_trainTargetOrig, m_trainLabelOrig, m_testOrig, m_testTargetOrig, m_testLabelOrig, m_nTrain, m_nTest, m_nClass, m_nDomain, m_nFeatures, m_positiveTarget, m_negativeTarget );
    }
    else if ( name == "BREAST" ) // read breast-cancer dataset
    {
        DatasetReader r;
        // call by reference, memory is allcated in the DatasetReader
        r.readBREASTCANCERWISCONSIN ( m_datasetPath+"/"+m_dataPath, m_trainOrig, m_trainTargetOrig, m_trainLabelOrig, m_testOrig, m_testTargetOrig, m_testLabelOrig, m_nTrain, m_nTest, m_nClass, m_nDomain, m_nFeatures, m_positiveTarget, m_negativeTarget );
    }
    else if ( name == "CREDIT" ) // read australian-credit dataset
    {
        DatasetReader r;
        // call by reference, memory is allcated in the DatasetReader
        r.readAUSTRALIANCREDIT ( m_datasetPath+"/"+m_dataPath, m_trainOrig, m_trainTargetOrig, m_trainLabelOrig, m_testOrig, m_testTargetOrig, m_testLabelOrig, m_nTrain, m_nTest, m_nClass, m_nDomain, m_nFeatures, m_positiveTarget, m_negativeTarget );
    }
    else if ( name == "DIABETES" ) // read diabetes dataset
    {
        DatasetReader r;
        // call by reference, memory is allcated in the DatasetReader
        r.readDIABETES ( m_datasetPath+"/"+m_dataPath, m_trainOrig, m_trainTargetOrig, m_trainLabelOrig, m_testOrig, m_testTargetOrig, m_testLabelOrig, m_nTrain, m_nTest, m_nClass, m_nDomain, m_nFeatures, m_positiveTarget, m_negativeTarget );
    }
    else if ( name == "GERMAN" ) // read german dataset
    {
        DatasetReader r;
        // call by reference, memory is allcated in the DatasetReader
        r.readGERMAN ( m_datasetPath+"/"+m_dataPath, m_trainOrig, m_trainTargetOrig, m_trainLabelOrig, m_testOrig, m_testTargetOrig, m_testLabelOrig, m_nTrain, m_nTest, m_nClass, m_nDomain, m_nFeatures, m_positiveTarget, m_negativeTarget );
    }
    else if ( name == "GLASS" ) // read glass dataset
    {
        DatasetReader r;
        // call by reference, memory is allcated in the DatasetReader
        r.readGLASS ( m_datasetPath+"/"+m_dataPath, m_trainOrig, m_trainTargetOrig, m_trainLabelOrig, m_testOrig, m_testTargetOrig, m_testLabelOrig, m_nTrain, m_nTest, m_nClass, m_nDomain, m_nFeatures, m_positiveTarget, m_negativeTarget );
    }
    else if ( name == "HEART-SPECTF" ) // read heart dataset
    {
        DatasetReader r;
        // call by reference, memory is allcated in the DatasetReader
        r.readHEART ( m_datasetPath+"/"+m_dataPath, m_trainOrig, m_trainTargetOrig, m_trainLabelOrig, m_testOrig, m_testTargetOrig, m_testLabelOrig, m_nTrain, m_nTest, m_nClass, m_nDomain, m_nFeatures, m_positiveTarget, m_negativeTarget );
    }
    else if ( name == "HEPATITIS" ) // read hepatitis dataset
    {
        DatasetReader r;
        // call by reference, memory is allcated in the DatasetReader
        r.readHEPATITIS ( m_datasetPath+"/"+m_dataPath, m_trainOrig, m_trainTargetOrig, m_trainLabelOrig, m_testOrig, m_testTargetOrig, m_testLabelOrig, m_nTrain, m_nTest, m_nClass, m_nDomain, m_nFeatures, m_positiveTarget, m_negativeTarget );
    }
    else if ( name == "IONOSPHERE" ) // read ionophsere dataset
    {
        DatasetReader r;
        // call by reference, memory is allcated in the DatasetReader
        r.readIONOSPHERE ( m_datasetPath+"/"+m_dataPath, m_trainOrig, m_trainTargetOrig, m_trainLabelOrig, m_testOrig, m_testTargetOrig, m_testLabelOrig, m_nTrain, m_nTest, m_nClass, m_nDomain, m_nFeatures, m_positiveTarget, m_negativeTarget );
    }
    else if ( name == "IRIS" ) // read iris dataset
    {
        DatasetReader r;
        // call by reference, memory is allcated in the DatasetReader
        r.readIRIS ( m_datasetPath+"/"+m_dataPath, m_trainOrig, m_trainTargetOrig, m_trainLabelOrig, m_testOrig, m_testTargetOrig, m_testLabelOrig, m_nTrain, m_nTest, m_nClass, m_nDomain, m_nFeatures, m_positiveTarget, m_negativeTarget );
    }
    else if ( name == "LETTER" ) // read letter dataset
    {
        DatasetReader r;
        // call by reference, memory is allcated in the DatasetReader
        r.readLETTER ( m_datasetPath+"/"+m_dataPath, m_trainOrig, m_trainTargetOrig, m_trainLabelOrig, m_testOrig, m_testTargetOrig, m_testLabelOrig, m_nTrain, m_nTest, m_nClass, m_nDomain, m_nFeatures, m_positiveTarget, m_negativeTarget );
    }
    else if ( name == "MONKS-1" ) // read monks1 dataset
    {
        DatasetReader r;
        // call by reference, memory is allcated in the DatasetReader
        r.readMONKS1 ( m_datasetPath+"/"+m_dataPath, m_trainOrig, m_trainTargetOrig, m_trainLabelOrig, m_testOrig, m_testTargetOrig, m_testLabelOrig, m_nTrain, m_nTest, m_nClass, m_nDomain, m_nFeatures, m_positiveTarget, m_negativeTarget );
    }
    else if ( name == "MONKS-2" ) // read monks2 dataset
    {
        DatasetReader r;
        // call by reference, memory is allcated in the DatasetReader
        r.readMONKS2 ( m_datasetPath+"/"+m_dataPath, m_trainOrig, m_trainTargetOrig, m_trainLabelOrig, m_testOrig, m_testTargetOrig, m_testLabelOrig, m_nTrain, m_nTest, m_nClass, m_nDomain, m_nFeatures, m_positiveTarget, m_negativeTarget );
    }
    else if ( name == "MONKS-3" ) // read monks3 dataset
    {
        DatasetReader r;
        // call by reference, memory is allcated in the DatasetReader
        r.readMONKS3 ( m_datasetPath+"/"+m_dataPath, m_trainOrig, m_trainTargetOrig, m_trainLabelOrig, m_testOrig, m_testTargetOrig, m_testLabelOrig, m_nTrain, m_nTest, m_nClass, m_nDomain, m_nFeatures, m_positiveTarget, m_negativeTarget );
    }
    else if ( name == "MUSHROOM" ) // read mushroom dataset
    {
        DatasetReader r;
        // call by reference, memory is allcated in the DatasetReader
        r.readMUSHROOM ( m_datasetPath+"/"+m_dataPath, m_trainOrig, m_trainTargetOrig, m_trainLabelOrig, m_testOrig, m_testTargetOrig, m_testLabelOrig, m_nTrain, m_nTest, m_nClass, m_nDomain, m_nFeatures, m_positiveTarget, m_negativeTarget );
    }
    else if ( name == "SATIMAGE" ) // read satimage dataset
    {
        DatasetReader r;
        // call by reference, memory is allcated in the DatasetReader
        r.readSATIMAGE ( m_datasetPath+"/"+m_dataPath, m_trainOrig, m_trainTargetOrig, m_trainLabelOrig, m_testOrig, m_testTargetOrig, m_testLabelOrig, m_nTrain, m_nTest, m_nClass, m_nDomain, m_nFeatures, m_positiveTarget, m_negativeTarget );
    }
    else if ( name == "SEGMENTATION" ) // read segmentation dataset
    {
        DatasetReader r;
        // call by reference, memory is allcated in the DatasetReader
        r.readSEGMENTATION ( m_datasetPath+"/"+m_dataPath, m_trainOrig, m_trainTargetOrig, m_trainLabelOrig, m_testOrig, m_testTargetOrig, m_testLabelOrig, m_nTrain, m_nTest, m_nClass, m_nDomain, m_nFeatures, m_positiveTarget, m_negativeTarget );
    }
    else if ( name == "SONAR" ) // read sonar dataset
    {
        DatasetReader r;
        // call by reference, memory is allcated in the DatasetReader
        r.readSONAR ( m_datasetPath+"/"+m_dataPath, m_trainOrig, m_trainTargetOrig, m_trainLabelOrig, m_testOrig, m_testTargetOrig, m_testLabelOrig, m_nTrain, m_nTest, m_nClass, m_nDomain, m_nFeatures, m_positiveTarget, m_negativeTarget );
    }
    else if ( name == "VEHICLE" ) // read vehicle dataset
    {
        DatasetReader r;
        // call by reference, memory is allcated in the DatasetReader
        r.readVEHICLE ( m_datasetPath+"/"+m_dataPath, m_trainOrig, m_trainTargetOrig, m_trainLabelOrig, m_testOrig, m_testTargetOrig, m_testLabelOrig, m_nTrain, m_nTest, m_nClass, m_nDomain, m_nFeatures, m_positiveTarget, m_negativeTarget );
    }
    else if ( name == "VOTES" ) // read votes dataset
    {
        DatasetReader r;
        // call by reference, memory is allcated in the DatasetReader
        r.readVOTES ( m_datasetPath+"/"+m_dataPath, m_trainOrig, m_trainTargetOrig, m_trainLabelOrig, m_testOrig, m_testTargetOrig, m_testLabelOrig, m_nTrain, m_nTest, m_nClass, m_nDomain, m_nFeatures, m_positiveTarget, m_negativeTarget );
    }
    else if ( name == "WINE" ) // read wine dataset
    {
        DatasetReader r;
        // call by reference, memory is allcated in the DatasetReader
        r.readWINE ( m_datasetPath+"/"+m_dataPath, m_trainOrig, m_trainTargetOrig, m_trainLabelOrig, m_testOrig, m_testTargetOrig, m_testLabelOrig, m_nTrain, m_nTest, m_nClass, m_nDomain, m_nFeatures, m_positiveTarget, m_negativeTarget );
    }
    else if ( name == "POKER" ) // read poker dataset
    {
        DatasetReader r;
        // call by reference, memory is allcated in the DatasetReader
        r.readPOKER ( m_datasetPath+"/"+m_dataPath, m_trainOrig, m_trainTargetOrig, m_trainLabelOrig, m_testOrig, m_testTargetOrig, m_testLabelOrig, m_nTrain, m_nTest, m_nClass, m_nDomain, m_nFeatures, m_positiveTarget, m_negativeTarget );
    }
    else if ( name == "YEAST" ) // read yeast dataset
    {
        DatasetReader r;
        // call by reference, memory is allcated in the DatasetReader
        r.readYEAST ( m_datasetPath+"/"+m_dataPath, m_trainOrig, m_trainTargetOrig, m_trainLabelOrig, m_testOrig, m_testTargetOrig, m_testLabelOrig, m_nTrain, m_nTest, m_nClass, m_nDomain, m_nFeatures, m_positiveTarget, m_negativeTarget );
    }
    else if ( name == "SURVIVAL" ) // read survival dataset
    {
        DatasetReader r;
        // call by reference, memory is allcated in the DatasetReader
        r.readSURVIVAL ( m_datasetPath+"/"+m_dataPath, m_trainOrig, m_trainTargetOrig, m_trainLabelOrig, m_testOrig, m_testTargetOrig, m_testLabelOrig, m_nTrain, m_nTest, m_nClass, m_nDomain, m_nFeatures, m_positiveTarget, m_negativeTarget );
    }
    else if ( name == "SPIDER" ) // read (generated by)spider dataset
    {
        DatasetReader r;
        // call by reference, memory is allcated in the DatasetReader
        r.readSPIDER ( m_datasetPath+"/"+m_dataPath, m_trainOrig, m_trainTargetOrig, m_trainLabelOrig, m_testOrig, m_testTargetOrig, m_testLabelOrig, m_nTrain, m_nTest, m_nClass, m_nDomain, m_nFeatures, m_positiveTarget, m_negativeTarget );
    }
    else
    {
        cout<<"Dataset not found:"<<name<<endl;
        exit ( 0 );
    }

    if(m_addConstantInput)
        addConstantInput();
    
    // reduce the size of the training set
    reduceTrainingSetSize ( m_subsampleTrainSet );

    // reduce the size of the features in the training set
    int nFeatOrig = m_nFeatures;
    reduceFeatureSize ( m_trainOrig, m_nTrain, m_nFeatures, m_subsampleFeatures, Framework::getFrameworkMode() );
    reduceFeatureSize ( m_testOrig, m_nTest, nFeatOrig, m_subsampleFeatures, true );

    // feature selection, based on a linear model
    if ( m_featureSelectionWriteBinaryDataset )
    {
        makeBinaryDataset();
        exit ( 0 );
    }

    // mix train features and labels
    mixDataset();
}

void Data::doBootstrapSampling ( REAL* probs, REAL* &train, REAL* &target, REAL* &targetEff, REAL* &targetRes, int* &label, int nTrainNew )
{
    cout<<endl<<"Do boostrap sampling of the dataset (size:"<<m_nTrain<<")"<<endl;
    cout<<"Random seed:"<<m_randomSeedBagging<<endl;
    srand ( m_randomSeedBagging );

    if ( nTrainNew > 0 && nTrainNew < m_nTrain )
        cout<<"Draw not a boostrap sample, make a simple random subset ("<<100.0* ( double ) nTrainNew/ ( double ) m_nTrain<<"%)"<<endl;

    REAL* trainNew = 0, *ptr0, *ptr1;
    if ( train )
        trainNew = new REAL[m_nFeatures*m_nTrain];
    REAL* targetNew = 0;
    if ( target )
        targetNew = new REAL[m_nClass*m_nDomain*m_nTrain];
    REAL* targetEffNew = 0;
    if ( targetEff )
        targetEffNew = new REAL[m_nClass*m_nDomain*m_nTrain];
    REAL* targetResNew = 0;
    if ( targetRes )
        targetResNew = new REAL[m_nClass*m_nDomain*m_nTrain];
    int* labelNew = 0;
    if ( Framework::getDatasetType() ==true )
        labelNew = new int[m_nDomain*m_nTrain];
    int* replicateCnt = new int[m_nTrain];
    for ( int i=0;i<m_nTrain;i++ )
        replicateCnt[i] = 0;

    int sampleCnt = 0;
    while ( ( sampleCnt < m_nTrain && nTrainNew == 0 ) || ( sampleCnt < nTrainNew && nTrainNew > 0 && nTrainNew < m_nTrain ) )
        //for(int i=0;i<m_nTrain;i++)
    {
        // random index
        int ind;
        if ( nTrainNew == 0 || nTrainNew >= m_nTrain ) // boostrap sample
        {
            if ( probs == 0 )
                ind = rand() %m_nTrain;
            else
                ind = vectorSampling ( probs, m_nTrain );
        }
        else  // random subset
        {
            ind = rand() %m_nTrain;
            while ( replicateCnt[ind] )
                ind = rand() %m_nTrain;
        }
        replicateCnt[ind]++;

        // train features
        if ( train )
        {
            ptr0 = train + ind * m_nFeatures;
            ptr1 = trainNew + sampleCnt * m_nFeatures;
            for ( int j=0;j<m_nFeatures;j++ )
                ptr1[j] = ptr0[j];
        }

        // targets
        if ( target )
        {
            ptr0 = target + ind * m_nClass*m_nDomain;
            ptr1 = targetNew + sampleCnt * m_nClass*m_nDomain;
            for ( int j=0;j<m_nClass*m_nDomain;j++ )
                ptr1[j] = ptr0[j];
        }

        // effects
        if ( targetEff )
        {
            ptr0 = targetEff + ind * m_nClass*m_nDomain;
            ptr1 = targetEffNew + sampleCnt * m_nClass*m_nDomain;
            for ( int j=0;j<m_nClass*m_nDomain;j++ )
                ptr1[j] = ptr0[j];
        }

        // residual
        if ( targetRes )
        {
            ptr0 = targetRes + ind * m_nClass*m_nDomain;
            ptr1 = targetResNew + sampleCnt * m_nClass*m_nDomain;
            for ( int j=0;j<m_nClass*m_nDomain;j++ )
                ptr1[j] = ptr0[j];
        }

        // train label
        if ( Framework::getDatasetType() ==true )
            for ( int d=0;d<m_nDomain;d++ )
                labelNew[d+sampleCnt*m_nDomain] = label[d+ind*m_nDomain];

        sampleCnt++;
    }

    int nonReplicates = 0, notUsed = 0, replicates = 0;
    for ( int i=0;i<m_nTrain;i++ )
    {
        if ( replicateCnt[i] == 0 )
            notUsed++;
        if ( replicateCnt[i] == 1 )
            nonReplicates++;
        if ( replicateCnt[i] > 1 )
            replicates++;
    }
    cout<<"notUsed:"<<notUsed<<" nonReplicates:"<<nonReplicates<<" replicates:"<<replicates;
    cout<<" ("<<100.0* ( REAL ) ( nonReplicates+replicates ) / ( REAL ) m_nTrain<<"%)"<<endl<<endl;

    delete[] replicateCnt;

    // set new data
    train = trainNew;
    target = targetNew;
    targetEff = targetEffNew;
    targetRes = targetResNew;
    label = labelNew;
}

int Data::vectorSampling ( REAL* probs, int length )
{
    double sum = 0.0;
    for ( int i=0;i<length;i++ )
        sum += probs[i];

    double value = sum * ( ( double ) rand() / ( double ) RAND_MAX );

    sum = 0.0;
    for ( int i=0;i<length;i++ )
    {
        sum += probs[i];
        if ( sum >= value )
            return i;
    }
    cout<<"value:"<<value<<endl<<"length:"<<length<<endl<<"sum:"<<sum<<endl;
    for ( int i=0;i<length;i++ )
        cout<<probs[i]<<" "<<flush;
    assert ( false );
    return -1;
}

void Data::makeBinaryDataset()
{
    cout<<endl;
    cout<<"Make binary dataset from selected features"<<endl;
    cout<<"Open features:"<<FEATURE_TXT_FILE<<endl;

    // read features from txt file
    fstream f;
    vector<int> features;
    f.open ( FEATURE_TXT_FILE,ios::in );
    if ( f.is_open() ==false )
        assert ( false );
    int value, nValidFeatures = 0;
    while ( f>>value )
        features.push_back ( value );
    f.close();

    // check featureIDs
    for ( int j=0;j<features.size();j++ )
        if ( features[j] >= m_nFeatures || features[j] == -1 )
            assert ( false );
        else
            nValidFeatures++;

    cout<<"nValidFeatures:"<<nValidFeatures<<endl;
    REAL* feat;
    int* label, N;

    if ( Framework::getFrameworkMode() == 1 )
    {
        cout<<"Write: binary.test"<<endl;
        f.open ( "binary.test", ios::out );
        feat = m_testOrig;
        label = m_testLabelOrig;
        N = m_nTest;
    }
    else
    {
        cout<<"Write: binary.train"<<endl;
        f.open ( "binary.train", ios::out );
        feat = m_trainOrig;
        label = m_trainLabelOrig;
        N = m_nTrain;
    }

    cout<<"#lines:"<<N<<endl;

    // dataset bounds
    f.write ( ( char* ) &N, sizeof ( int ) );
    f.write ( ( char* ) &m_nClass, sizeof ( int ) );
    f.write ( ( char* ) &m_nDomain, sizeof ( int ) );
    f.write ( ( char* ) &nValidFeatures, sizeof ( int ) );

    // write features
    for ( int i=0;i<N;i++ )
        for ( int j=0;j<features.size();j++ )
            f.write ( ( char* ) & ( feat[i*m_nFeatures + features[j]] ), sizeof ( REAL ) );

    // write labels
    f.write ( ( char* ) label, sizeof ( int ) *N*m_nDomain );
    f.close();

}

void Data::mixDataset()
{
    if ( m_nTrain )
    {
        m_mixDatasetIndices = new int[m_nTrain];
        for ( int i=0;i<m_nTrain;i++ )
            m_mixDatasetIndices[i] = i;
    }
    else
    {
        cout<<"Do no mix the dataset."<<endl;
        m_mixDatasetIndices = 0;
        return;
    }
    cout<<"Randomize the dataset: "<<m_nMixDataset*m_nTrain<<" line swaps [";

    int progress = m_nTrain*m_nMixDataset/10 + 1;
    REAL* tmp0 = new REAL[m_nFeatures];
    REAL* tmp1 = new REAL[m_nClass*m_nDomain];
    for ( int i=0;i<m_nTrain*m_nMixDataset;i++ )
    {
        if ( i%progress==0 )
            cout<<"."<<flush;

        // random index swaps
        int ind0 = rand() %m_nTrain;
        int ind1 = rand() %m_nTrain;

        // train features (REAL*)
        REAL* ptr0 = m_trainOrig + ind0 * m_nFeatures;
        REAL* ptr1 = m_trainOrig + ind1 * m_nFeatures;
        for ( int j=0;j<m_nFeatures;j++ )
        {
            tmp0[j] = ptr0[j];
            ptr0[j] = ptr1[j];
            ptr1[j] = tmp0[j];
        }

        // train targets (REAL*)
        ptr0 = m_trainTargetOrig + ind0 * m_nClass * m_nDomain;
        ptr1 = m_trainTargetOrig + ind1 * m_nClass * m_nDomain;
        for ( int j=0;j<m_nClass*m_nDomain;j++ )
        {
            tmp1[j] = ptr0[j];
            ptr0[j] = ptr1[j];
            ptr1[j] = tmp1[j];
        }

        // train label
        if ( Framework::getDatasetType() ==true )
        {
            for ( int d=0;d<m_nDomain;d++ )
            {
                int tmp = m_trainLabelOrig[d+ind0*m_nDomain];
                m_trainLabelOrig[d+ind0*m_nDomain] = m_trainLabelOrig[d+ind1*m_nDomain];
                m_trainLabelOrig[d+ind1*m_nDomain] = tmp;
            }
        }

        // index
        int tmp = m_mixDatasetIndices[ind0];
        m_mixDatasetIndices[ind0] = m_mixDatasetIndices[ind1];
        m_mixDatasetIndices[ind1] = tmp;
    }
    if ( tmp0 )
        delete[] tmp0;
    tmp0 = 0;
    if ( tmp1 )
        delete[] tmp1;
    tmp1 = 0;

    cout<<"] "<<"mixInd[0]:"<<m_mixDatasetIndices[0]<<"  mixInd["<<m_nTrain-1<<"]:"<<m_mixDatasetIndices[m_nTrain-1]<<endl;
}

void Data::loadNormalization ( int nCascade )
{
    // load normalization
    char buf[1024];
    sprintf ( buf,"%s/%s/normalization.dat.add%d",m_datasetPath.c_str(), m_tempPath.c_str(), nCascade );
    cout<<"Load mean and std: "<<buf<<endl;
    fstream f ( buf, ios::in );
    if ( f.is_open() == false )
        assert ( false );
    int n;
    f.read ( ( char* ) &n, sizeof ( int ) );
    if ( m_mean == 0 )
        m_mean = new REAL[n];
    if ( m_std == 0 )
        m_std = new REAL[n];
    f.read ( ( char* ) m_mean, sizeof ( REAL ) *n );
    f.read ( ( char* ) m_std, sizeof ( REAL ) *n );
    REAL min = 1e10, max = -1e10;
    for ( int i=0;i<n;i++ )
    {
        if ( min > m_mean[i] )
            min = m_mean[i];
        if ( max < m_mean[i] )
            max = m_mean[i];
    }
    cout<<"Mean:  min|max:"<<min<<"|"<<max<<endl;
    min = 1e10;
    max = -1e10;
    for ( int i=0;i<n;i++ )
    {
        if ( min > m_std[i] )
            min = m_std[i];
        if ( max < m_std[i] )
            max = m_std[i];
    }
    cout<<"Std:  min|max:"<<min<<"|"<<max<<endl;
    f.close();
}

void Data::allocMemForCrossValidationSets()
{
    cout<<"Alloc mem for cross validation data sets"<<endl;
    m_mean = new REAL[m_nFeatures];
    m_std = new REAL[m_nFeatures];

    if(m_validationType == "ValidationSet")
        m_nCross = 0;
    else
    {
        // partitioning to nCross-validation sets
        if ( m_nCross > m_nTrain )
        {
            cout<<"Limit: nCross=nTrain"<<endl;
            m_nCross = m_nTrain;
        }
        cout<<"Cross-validation settings: "<<m_nCross<<" sets"<<endl;
    }
    
    // calc global mean and standard deviation over whole dataset
    cout<<"Calculating mean and std per input"<<endl;
    double minStd = 1e10, maxStd = -1e10, minMean = 1e10, maxMean = -1e10, minValue = 1e10, maxValue = -1e10;
    for ( int i=0;i<m_nFeatures;i++ )
    {
        // calc mean
        double mean = 0.0;
        for ( int j=0;j<m_nTrain;j++ )
        {
            REAL v = m_trainOrig[j*m_nFeatures + i];
            mean += v;
            if ( minValue > v )
                minValue = v;
            if ( maxValue < v )
                maxValue = v;
        }
        mean /= ( double ) m_nTrain;

        // calc standard deviation
        double std = 0.0;
        for ( int j=0;j<m_nTrain;j++ )
            std += ( mean - m_trainOrig[j*m_nFeatures + i] ) * ( mean - m_trainOrig[j*m_nFeatures + i] );
        std = sqrt ( std/ ( double ) ( m_nTrain-1 ) );

        if ( m_datasetName=="KDDCup09Large" || m_datasetName=="KDDCup09Small" ) // || m_datasetName=="BINARY")
        {
            double max = -1e10;
            for ( int j=0;j<m_nTrain;j++ )
                if ( max < fabs ( m_trainOrig[j*m_nFeatures + i]-mean ) )
                    max = fabs ( m_trainOrig[j*m_nFeatures + i]-mean );
            std = max;
        }

        if ( fabs ( std ) < 1e-9 && mean == 0.0 ) // constant zero input
        {
            //cout<<"Feature nr:"<<i<<" is constant zero (mean:"<<mean<<"), set std=1e10"<<endl;
            cout<<"f:"<<i<<"=0 "<<flush;
            std = 1e10;
        }
        if ( fabs ( std ) < 1e-9 && mean != 0.0 ) // constant input
        {
            //cout<<"Feature nr:"<<i<<" is constant (mean:"<<mean<<"), set std="<<mean<<" and mean=0"<<endl;
            cout<<"f:"<<i<<"=c "<<flush;
            std = mean;
            mean = 0.0;
        }
        if ( mean==1.0 ) // constant one input
        {
            //cout<<"Feature nr:"<<i<<" mean=1, set std=1 and mean=0"<<endl;
            cout<<"f:"<<i<<"=1 "<<flush;
            std = 1.0;
            mean = 0.0;
        }
        if ( std < m_standardDeviationMin ) // limit to a small positive value
        {
            //cout<<"Feature nr:"<<i<<" "<<"("<<std<<") is limited in std="<<m_standardDeviationMin<<endl;
            cout<<"f:"<<i<<"lim "<<flush;
            std = m_standardDeviationMin;
        }

        minStd = minStd > std? std : minStd;
        maxStd = maxStd < std? std : maxStd;
        minMean = minMean > mean? mean : minMean;
        maxMean = maxMean < mean? mean : maxMean;

        // save them
        m_mean[i] = mean;
        m_std[i] = std;
    }
    if ( m_enableStaticNormalization )
    {
        cout<<"Static mean:"<<m_staticMeanNormalization<<" and std:"<<m_staticStdNormalization<<endl;
        for ( int i=0;i<m_nFeatures;i++ )
        {
            m_mean[i] = m_staticMeanNormalization;
            m_std[i] = m_staticStdNormalization;
        }
        minMean = m_staticMeanNormalization;
        maxMean = m_staticMeanNormalization;
        minStd = m_staticStdNormalization;
        maxStd = m_staticStdNormalization;
    }
    if ( m_enableGlobalMeanStdEstimate )
    {
        cout<<"Calc average of mean and std"<<endl;
        double mean = 0.0;
        for ( int i=0;i<m_nFeatures;i++ )
            mean += m_mean[i];
        mean /= ( double ) m_nFeatures;
        for ( int i=0;i<m_nFeatures;i++ )
            m_mean[i] = mean;
        minMean = maxMean = mean;

        double std = 0.0;
        int stdCnt = 0;
        for ( int i=0;i<m_nFeatures;i++ )
        {
            if ( m_std[i] != 1e10 )
            {
                std += m_std[i];
                stdCnt++;
            }
        }
        if ( stdCnt == 0 )
            assert ( false );
        std /= ( double ) stdCnt;
        for ( int i=0;i<m_nFeatures;i++ )
            m_std[i] = std;
        minStd = maxStd = std;
    }
    if ( m_enableProbablisticNormalization )
    {
        cout<<"Calc probablistic normalization"<<endl;
        minStd = 1e10;
        maxStd = -1e10;
        minMean = 1e10;
        maxMean = -1e10;
        for ( int i=0;i<m_nFeatures;i++ )
        {
            REAL min = 1e10, max = -1e10;
            for ( int j=0;j<m_nTrain;j++ )
            {
                REAL v = m_trainOrig[i + j*m_nFeatures];
                if ( min > v )
                    min = v;
                if ( max < v )
                    max = v;
            }
            REAL diff = max - min;
            m_mean[i] = min;
            m_std[i] = diff;
            if ( m_std[i] < 1e-6 )
                m_std[i] = 1.0;

            minStd = minStd > m_std[i]? m_std[i] : minStd;
            maxStd = maxStd < m_std[i]? m_std[i] : maxStd;
            minMean = minMean > m_mean[i]? m_mean[i] : minMean;
            maxMean = maxMean < m_mean[i]? m_mean[i] : maxMean;
        }
        cout<<"mean|std:"<<endl;
        for ( int i=0;i<m_nFeatures;i++ )
            cout<<m_mean[i]<<"|"<<m_std[i]<<" ";
        cout<<endl;
    }
    cout<<"Min|Max mean: "<<minMean<<"|"<<maxMean<<"   Min|Max std: "<<minStd<<"|"<<maxStd<<"   Min|Max value: "<<minValue<<"|"<<maxValue<<endl;

    // target means
    cout<<"Target means: "<<flush;
    for ( int i=0;i<m_nClass*m_nDomain;i++ )
    {
        double mean = 0.0;
        REAL* ptr = m_trainTargetOrig + i * m_nClass * m_nDomain;
        for ( int j=0;j<m_nTrain;j++ )
            mean += ptr[j];
        cout<<i<<":"<<mean/ ( double ) ( m_nTrain ) <<" ";
    }
    cout<<endl;

    // save normalization
    char buf[1024];
    sprintf ( buf,"%s/%s/normalization.dat.add%d",m_datasetPath.c_str(), m_tempPath.c_str(), m_nCascadeInputs );
    cout<<"Save mean and std: "<<buf<<endl;
    fstream f ( buf, ios::out );
    f.write ( ( char* ) &m_nFeatures, sizeof ( int ) );
    f.write ( ( char* ) m_mean, sizeof ( REAL ) *m_nFeatures );
    f.write ( ( char* ) m_std, sizeof ( REAL ) *m_nFeatures );
    f.close();

    m_mixList = new int[m_nTrain];

    // mixing list
    for ( int i=0;i<m_nTrain;i++ )
        m_mixList[i] = i;

    // fix the randomness
    cout<<"Random seed:"<<m_randSeed<<endl;
    srand ( m_randSeed );

    cout<<"nFeatures:"<<m_nFeatures<<endl;
    cout<<"nClass:"<<m_nClass<<endl;
    cout<<"nDomain:"<<m_nDomain<<endl;

    if ( m_validationType == "ValidationSet" )
    {
        // no cross validation set
        m_trainSize = new int[1];
        m_trainSize[0] = m_nTrain;
        return;
    }
    
    
    m_trainTargetOrigEffect = new REAL[m_nClass*m_nDomain*m_nTrain];
    m_trainTargetOrigResidual = new REAL[m_nClass*m_nDomain*m_nTrain];

    // allocate mem for cross validation sets
    m_trainSize = new int[m_nCross+1];
    m_train = new REAL*[m_nCross+1];
    m_trainTarget = new REAL*[m_nCross+1];
    m_trainTargetEffect = new REAL*[m_nCross+1];
    m_trainTargetResidual = new REAL*[m_nCross+1];
    m_trainLabel = new int*[m_nCross+1];
    if(m_validationType == "Bagging")
        m_trainBaggingIndex = new int*[m_nCross+1];

    m_probeSize = new int[m_nCross+1];
    m_probe = new REAL*[m_nCross+1];
    m_probeTarget = new REAL*[m_nCross+1];
    m_probeTargetEffect = new REAL*[m_nCross+1];
    m_probeTargetResidual = new REAL*[m_nCross+1];
    m_probeLabel = new int*[m_nCross+1];
    m_probeIndex = new int*[m_nCross+1];

    
    // make a randomized index list (by random index swaps)
    int index0, index1, tmp;
    cout<<"Make "<<m_nTrain*m_nMixTrainList<<" index swaps (randomize sample index list)"<<endl;
    for ( int i=0;i<m_nTrain*m_nMixTrainList;i++ )
    {
        index0 = rand() % m_nTrain;
        index1 = rand() % m_nTrain;

        // swap
        tmp = m_mixList[index0];
        m_mixList[index0] = m_mixList[index1];
        m_mixList[index1] = tmp;
    }

    if( m_validationType == "Retraining" || m_validationType == "CrossFoldMean" )
    {
        m_slotBoundaries = new int[m_nCross+2];
    
        double partitionSize = ( double ) m_nTrain / ( double ) m_nCross;
        double accumulatedSize = partitionSize;
        int cnt = 0, currentSize = -1;
        m_slotBoundaries[0] = 0;
        m_slotBoundaries[m_nCross+1] = m_nTrain;
        cout<<"partition size: "<<partitionSize<<endl;
    
        // calculate train + probe size
        for ( int i=0;i<=m_nTrain;i++ )
        {
            currentSize++;
            if ( cnt < m_nCross )
            {
                if ( i == ( int ) round ( accumulatedSize ) || i==m_nTrain )
                {
                    m_slotBoundaries[cnt+1] = i;
                    m_probeSize[cnt] = currentSize;
                    m_trainSize[cnt] = m_nTrain - currentSize;
                    currentSize = 0;
                    accumulatedSize += partitionSize;
                    cnt++;
                }
            }
        }
        m_trainSize[m_nCross] = m_nTrain;  // retraining set
        m_probeSize[m_nCross] = 0;
        
        // print splits
        int sum = 0;
        cout<<"slot: TRAIN | PROBE"<<endl<<"==================="<<endl;
        for ( int i=0;i<m_nCross+1;i++ )
        {
            cout<<i<<": "<<m_trainSize[i]<<" | "<<m_probeSize[i]<<endl;
            sum += m_probeSize[i];
        }
        cout<<"probe sum:"<<sum<<endl;
    }
    else if ( m_validationType == "Bagging" )
    {
        bool* bagSamples = new bool[m_nTrain];
        cout<<"Bagging sizes: TRAIN | PROBE"<<endl<<"============================"<<endl;
        for(int i=0;i<m_nCross;i++)
        {
            m_trainBaggingIndex[i] = new int[m_nTrain];
            
            // simulate boostrap sampling: sampling with replacenent
            srand(Framework::getRandomSeed() + i);
            int cnt = 0;
            for(int j=0;j<m_nTrain;j++)
                bagSamples[j] = 0;
            for(int j=0;j<m_nTrain;j++)
            {
                int ind = rand() % m_nTrain;
                bagSamples[ind] = 1;
                m_trainBaggingIndex[i][j] = ind;
            }
            for(int j=0;j<m_nTrain;j++)
                cnt += bagSamples[j];
            m_trainSize[i] = m_nTrain;
            m_probeSize[i] = m_nTrain - cnt;
            
            m_probeIndex[i] = new int[m_probeSize[i]];
            cnt = 0;
            for(int j=0;j<m_nTrain;j++)
            {
                if(bagSamples[j] == false)
                {
                    m_probeIndex[i][cnt] = j;
                    cnt++;
                }
            }
            cout<<i<<": "<<m_trainSize[i]<<" | "<<m_probeSize[i]<<"  ("<<100.0*(double)m_probeSize[i]/(double)m_nTrain<<"% in probe)"<<endl;
        }
        m_trainSize[m_nCross] = 0;
        m_probeSize[m_nCross] = 0;
        m_probeIndex[m_nCross] = 0;
        m_trainBaggingIndex[m_nCross] = 0;
        delete[] bagSamples;
        
        // make a summary (#zeros, mean coverage)
        int* bagCnt = new int[m_nTrain];
        for(int i=0;i<m_nTrain;i++)
            bagCnt[i] = 0;
        for(int i=0;i<m_nCross;i++)
            for(int j=0;j<m_nTrain;j++)
                bagCnt[m_trainBaggingIndex[i][j]]++;
        cout<<"Bagging summary: #averaged: and  #cnt"<<endl;
        for(int nr=0;nr<2*m_nCross;nr++)
        {
            int cnt = 0;
            for(int i=0;i<m_nTrain;i++)
                if(bagCnt[i] == nr)
                    cnt++;
            cout<<"n:"<<nr<<"|#"<<cnt<<" ";
        }
        cout<<endl;
        delete[] bagCnt;
    }
    else
        assert(false);
    
    // allocate mem + copy data to cross-validation slots
    for ( int i=0;i<m_nCross+1;i++ )
    {
        // allocate train mem
        int nTrain = m_trainSize[i];
        if ( m_enableSaveMemory == false )
            m_train[i] = new REAL[nTrain * m_nFeatures];
        else
            m_train[i] = 0;
        m_trainTarget[i] = new REAL[nTrain * m_nClass * m_nDomain];
        m_trainTargetEffect[i] = new REAL[nTrain * m_nClass * m_nDomain];
        m_trainTargetResidual[i] = new REAL[nTrain * m_nClass * m_nDomain];
        m_trainLabel[i] = new int[nTrain*m_nDomain];

        // allocate probe mem
        int nProbe = m_probeSize[i];
        if ( nProbe )
        {
            if ( m_enableSaveMemory == false )
                m_probe[i] = new REAL[nProbe * m_nFeatures];
            else
                m_probe[i] = 0;
            m_probeTarget[i] = new REAL[nProbe * m_nClass * m_nDomain];
            m_probeTargetEffect[i] = new REAL[nProbe * m_nClass * m_nDomain];
            m_probeTargetResidual[i] = new REAL[nProbe * m_nClass * m_nDomain];
            m_probeLabel[i] = new int[nProbe*m_nDomain];
            if ( m_validationType != "Bagging" )
                m_probeIndex[i] = new int[nProbe];
        }
        else
        {
            m_probe[i] = 0;
            m_probeTarget[i] = 0;
            m_probeTargetEffect[i] = 0;
            m_probeTargetResidual[i] = 0;
            m_probeLabel[i] = 0;
            m_probeIndex[i] = 0;
        }
    }

    // alloc index list
    m_crossIndex = new int[m_nTrain];
    for ( int i=0;i<m_nTrain;i++ )
        m_crossIndex[i] = -1;
    
}

void Data::readEffectFile()
{
    if(m_validationType == "ValidationSet")
        return;
    
    for ( int i=0;i<m_nClass*m_nDomain*m_nTrain;i++ )
        m_trainTargetOrigEffect[i] = 0.0;

    string name = m_datasetPath + "/" + m_fullPredPath + "/" + m_trainOnFullPredictorFile;
    fstream f ( name.c_str(), ios::in );
    if ( f.is_open() && m_trainOnFullPredictorFile!="" )
    {
        cout<<"Read fullPredictor:"<<name<<"  ";
        f.read ( ( char* ) m_trainTargetOrigEffect, sizeof ( REAL ) *m_nClass*m_nDomain*m_nTrain );

        double rmse0 = 0.0, rmse1 = 0.0, err;
        for ( int i=0;i<m_nClass*m_nDomain;i++ )
        {
            for ( int j=0;j<m_nTrain;j++ )
            {
                err = m_trainTargetOrigEffect[j*m_nClass*m_nDomain + i] - m_trainTargetOrig[j*m_nClass*m_nDomain + i];
                rmse0 += err * err;
            }
        }
        cout<<"RMSE:"<<sqrt ( rmse0/ ( double ) ( m_nClass*m_nDomain*m_nTrain ) ) <<"(retrain:"<<sqrt ( rmse1/ ( double ) ( m_nClass*m_nDomain*m_nTrain ) ) <<")"<<endl;

        f.close();
    }
    else
        cout<<"Can not open effect file:"<<name<<endl;

    // residual training: res = target - effect
    cout<<"Init residuals"<<endl;
    for ( int i=0;i<m_nClass*m_nDomain*m_nTrain;i++ )
        m_trainTargetOrigResidual[i] = m_trainTargetOrig[i] - m_trainTargetOrigEffect[i];
}

void Data::fillNCrossValidationSet ( int n )
{
    // alloc new memory
    if ( m_train[n] )
        delete[] m_train[n];
    m_train[n] = 0;
    m_train[n] = new REAL[m_trainSize[n]*m_nFeatures];
    for ( int i=0;i<m_trainSize[n]*m_nFeatures;i++ )
        m_train[n][i] = 0.0;
    if ( m_probe[n] )
        delete[] m_probe[n];
    m_probe[n] = 0;
    if ( m_probeSize[n] )
        m_probe[n] = new REAL[m_probeSize[n]*m_nFeatures];
    for ( int i=0;i<m_probeSize[n]*m_nFeatures;i++ )
        m_probe[n][i] = 0.0;

    if(m_validationType == "Bagging")
    {
        bool* bagSamples = new bool[m_nTrain];
        for(int i=0;i<m_nTrain;i++)
            bagSamples[i] = 0;
        for(int i=0;i<m_nTrain;i++)
        {
            int ind = m_trainBaggingIndex[n][i];
            bagSamples[ind] = 1;
            for(int j=0;j<m_nFeatures;j++)
                m_train[n][i*m_nFeatures+j] = m_trainOrig[ind*m_nFeatures + j];
        }
        int cnt = 0;
        for(int i=0;i<m_nTrain;i++)
        {
            if(bagSamples[i] == false)
            {
                for(int j=0;j<m_nFeatures;j++)
                    m_probe[n][cnt*m_nFeatures+j] = m_trainOrig[i*m_nFeatures + j];
                cnt++;
            }
        }
        if(cnt != m_probeSize[n])
        {
            cout<<"cnt:"<<cnt<<" probeSize"<<m_probeSize[n]<<endl;
            assert(false);
        }
        delete[] bagSamples;
    }
    else
    {
        // slot of probeset
        int begin = m_slotBoundaries[n];
        int end = m_slotBoundaries[n+1];
    
        int probeCnt = 0, trainCnt = 0;
    
        // go through whole trainOrig set
        for ( int j=0;j<m_nTrain;j++ )
        {
            int index = m_mixList[j];
    
            // probe set
            if ( j>=begin && j <end )
            {
                for ( int k=0;k<m_nFeatures;k++ )
                    m_probe[n][probeCnt*m_nFeatures + k] = m_trainOrig[index*m_nFeatures + k];
                probeCnt++;
            }
            else  // train set
            {
                for ( int k=0;k<m_nFeatures;k++ )
                    m_train[n][trainCnt*m_nFeatures + k] = m_trainOrig[index*m_nFeatures + k];
                trainCnt++;
            }
        }
    
        if ( probeCnt != m_probeSize[n] || trainCnt != m_trainSize[n] ) // safety check
            assert ( false );
    }
}

void Data::freeNCrossValidationSet ( int n )
{
    if ( m_train[n] )
        delete[] m_train[n];
    m_train[n] = 0;
    if ( m_probe[n] )
        delete[] m_probe[n];
    m_probe[n] = 0;
}

void Data::doFeatureSelection()
{
    bool* selectedFeatures = new bool[m_nFeatures];
    InputFeatureSelector::selectFeatures ( selectedFeatures, m_trainOrig, m_nFeatures, m_nTrain, m_trainLabelOrig, m_trainTargetOrigResidual, m_nClass, m_nDomain );

    delete[] selectedFeatures;
}

void Data::partitionDatasetToCrossValidationSets()
{
    cout<<"Partition dataset to cross validation sets"<<endl;

    // read the effect file
    readEffectFile();

    // write the first lines to a file
    if(m_trainOrig)
    { fstream f("Atrain.txt",ios::out); for ( int i=0;i<m_nTrain && i < 1000;i++ ){for ( int j=0;j<m_nFeatures;j++ ) f<<m_trainOrig[i*m_nFeatures + j]<<" ";f<<endl;}f.close();}
    if(m_testOrig)
    { fstream f("Atest.txt",ios::out); for ( int i=0;i<m_nTest && i < 1000;i++ ){for ( int j=0;j<m_nFeatures;j++ ) f<<m_testOrig[i*m_nFeatures + j]<<" ";f<<endl;}f.close();}
    if(m_valid)
    { fstream f("Avalid.txt",ios::out); for ( int i=0;i<m_validSize && i < 1000;i++ ){for ( int j=0;j<m_nFeatures;j++ ) f<<m_valid[i*m_nFeatures + j]<<" ";f<<endl;}f.close();}
    
    // apply mean and std to input features
    cout<<"Apply mean and std correction to train input features"<<endl;
    for ( int i=0;i<m_nTrain;i++ )
        for ( int j=0;j<m_nFeatures;j++ )
            m_trainOrig[i*m_nFeatures + j] = ( m_trainOrig[i*m_nFeatures + j] - m_mean[j] ) / m_std[j];

    // print min and max values in features
    REAL min = 1e10, max = -1e10;
    for ( int i=0;i<m_nTrain;i++ )
        for ( int j=0;j<m_nFeatures;j++ )
        {
            if ( min > m_trainOrig[i*m_nFeatures + j] )
                min = m_trainOrig[i*m_nFeatures + j];
            if ( max < m_trainOrig[i*m_nFeatures + j] )
                max = m_trainOrig[i*m_nFeatures + j];
        }
    cout<<"Min/Max after apply mean/std: "<<min<<"/"<<max<<endl;

    // print min and max values in targets
    min = 1e10;
    max = -1e10;
    m_targetMean = new REAL[m_nClass*m_nDomain];
    double* targetMean = new double[m_nClass*m_nDomain];
    for(int i=0;i<m_nClass*m_nDomain;i++)
        targetMean[i] = 0.0;
    for ( int i=0;i<m_nTrain;i++ )
        for ( int j=0;j<m_nClass*m_nDomain;j++ )
        {
            targetMean[j] += m_trainTargetOrig[i*m_nClass*m_nDomain + j];
            if ( min > m_trainTargetOrig[i*m_nClass*m_nDomain + j] )
                min = m_trainTargetOrig[i*m_nClass*m_nDomain + j];
            if ( max < m_trainTargetOrig[i*m_nClass*m_nDomain + j] )
                max = m_trainTargetOrig[i*m_nClass*m_nDomain + j];
        }
    for(int i=0;i<m_nClass*m_nDomain;i++)
        m_targetMean[i] = targetMean[i]/(double)m_nTrain;
    delete[] targetMean;
    
    cout<<"Min/Max target: "<<min<<"/"<<max<<endl<<"Mean target: ";
    for(int i=0;i<m_nClass*m_nDomain;i++)
        cout<<m_targetMean[i]<<" ";
    cout<<endl<<endl;

    if(m_validationType == "Retraining" || m_validationType == "CrossFoldMean")
    {
        int* labels = new int[m_nDomain];
    
        // copy data to cross-validation slots
        for ( int i=0;i<m_nCross+1;i++ )
        {
            // slot of probeset
            int begin = m_slotBoundaries[i];
            int end = m_slotBoundaries[i+1];
    
            int probeCnt = 0, trainCnt = 0;
    
            // go through whole trainOrig set
            for ( int j=0;j<m_nTrain;j++ )
            {
                int index = m_mixList[j];
                if ( Framework::getDatasetType() )
                {
                    for ( int d=0;d<m_nDomain;d++ )
                        labels[d] = m_trainLabelOrig[d+index*m_nDomain];
                }
    
                // probe set
                if ( j>=begin && j <end )
                {
                    m_probeIndex[i][probeCnt] = index;
                    for ( int d=0;d<m_nDomain;d++ )
                        m_probeLabel[i][d+probeCnt*m_nDomain] = labels[d];
                    for ( int k=0;k<m_nFeatures;k++ )
                        if ( m_enableSaveMemory == false )
                            m_probe[i][probeCnt*m_nFeatures + k] = m_trainOrig[index*m_nFeatures + k];
                    for ( int k=0;k<m_nClass*m_nDomain;k++ )
                    {
                        m_probeTarget[i][probeCnt*m_nClass*m_nDomain + k] = m_trainTargetOrig[index*m_nClass*m_nDomain + k];
                        m_probeTargetEffect[i][probeCnt*m_nClass*m_nDomain + k] = m_trainTargetOrigEffect[index*m_nClass*m_nDomain + k];
                        m_probeTargetResidual[i][probeCnt*m_nClass*m_nDomain + k] = m_trainTargetOrigResidual[index*m_nClass*m_nDomain + k];
                    }
                    probeCnt++;
                    m_crossIndex[j] = i;
                }
                else  // train set
                {
                    for ( int d=0;d<m_nDomain;d++ )
                        m_trainLabel[i][d+trainCnt*m_nDomain] = labels[d];
                    for ( int k=0;k<m_nFeatures;k++ )
                        if ( m_enableSaveMemory == false )
                            m_train[i][trainCnt*m_nFeatures + k] = m_trainOrig[index*m_nFeatures + k];
                    for ( int k=0;k<m_nClass*m_nDomain;k++ )
                    {
                        m_trainTarget[i][trainCnt*m_nClass*m_nDomain + k] = m_trainTargetOrig[index*m_nClass*m_nDomain + k];
                        m_trainTargetEffect[i][trainCnt*m_nClass*m_nDomain + k] = m_trainTargetOrigEffect[index*m_nClass*m_nDomain + k];
                        m_trainTargetResidual[i][trainCnt*m_nClass*m_nDomain + k] = m_trainTargetOrigResidual[index*m_nClass*m_nDomain + k];
                    }
                    trainCnt++;
                }
            }
            if ( probeCnt != m_probeSize[i] || trainCnt != m_trainSize[i] ) // safety check
                assert ( false );
        }
    
        if ( labels )
            delete[] labels;
    
        for ( int i=0;i<m_nTrain;i++ )
            if ( m_crossIndex[i] == -1 )
                assert ( false );
    }
    else if(m_validationType == "Bagging")
    {
        bool* bagSamples = new bool[m_nTrain];
        for ( int i=0;i<m_nCross;i++ )
        {
            // train sets
            for(int j=0;j<m_nTrain;j++)
                bagSamples[j] = 0;
            for(int j=0;j<m_nTrain;j++)
            {
                uint ind = m_trainBaggingIndex[i][j];
                bagSamples[ind] = 1;  // mark
                
                if ( Framework::getDatasetType() )
                    for ( int d=0;d<m_nDomain;d++ )
                        m_trainLabel[i][d+j*m_nDomain] = m_trainLabelOrig[d+ind*m_nDomain];
                for ( int k=0;k<m_nFeatures;k++ )
                    if ( m_enableSaveMemory == false )
                        m_train[i][j*m_nFeatures + k] = m_trainOrig[ind*m_nFeatures + k];
                for ( int k=0;k<m_nClass*m_nDomain;k++ )
                {
                    m_trainTarget[i][j*m_nClass*m_nDomain + k] = m_trainTargetOrig[ind*m_nClass*m_nDomain + k];
                    m_trainTargetEffect[i][j*m_nClass*m_nDomain + k] = m_trainTargetOrigEffect[ind*m_nClass*m_nDomain + k];
                    m_trainTargetResidual[i][j*m_nClass*m_nDomain + k] = m_trainTargetOrigResidual[ind*m_nClass*m_nDomain + k];
                }
            }
            
            // probe sets
            int cnt = 0;
            for(int j=0;j<m_nTrain;j++)
                cnt += bagSamples[j];
            if(m_nTrain - cnt != m_probeSize[i])
                assert(false);
            cnt = 0;
            for(int j=0;j<m_nTrain;j++)
            {
                if(bagSamples[j] == false)
                {
                    if ( Framework::getDatasetType() )
                        for ( int d=0;d<m_nDomain;d++ )
                            m_probeLabel[i][d+cnt*m_nDomain] = m_trainLabelOrig[d+j*m_nDomain];
                    for ( int k=0;k<m_nFeatures;k++ )
                        if ( m_enableSaveMemory == false )
                            m_probe[i][cnt*m_nFeatures + k] = m_trainOrig[j*m_nFeatures + k];
                    for ( int k=0;k<m_nClass*m_nDomain;k++ )
                    {
                        m_probeTarget[i][cnt*m_nClass*m_nDomain + k] = m_trainTargetOrig[j*m_nClass*m_nDomain + k];
                        m_probeTargetEffect[i][cnt*m_nClass*m_nDomain + k] = m_trainTargetOrigEffect[j*m_nClass*m_nDomain + k];
                        m_probeTargetResidual[i][cnt*m_nClass*m_nDomain + k] = m_trainTargetOrigResidual[j*m_nClass*m_nDomain + k];
                    }
                    cnt++;
                }
            }
            if(cnt != m_probeSize[i])
                assert(false);
        }
        delete[] bagSamples;
    }
    else if(m_validationType == "ValidationSet")
    {
        ;
    }
    else
        assert(false);
}

void Data::fillCascadeLearningInputs()
{
    cout<<endl<<"Add effects (predictions of previous algorithms) as inputs to dataset"<<endl;

    // load the fullPredictors
    vector<string> files = m_algorithmNameList; //Data::getDirectoryFileList(m_datasetPath + "/" + m_fullPredPath + "/");
    vector<string> m_usedFiles;

    for ( int i=0;i<files.size();i++ )
        if ( files[i].at ( files[i].size()-1 ) != '.' && files[i].find ( ".dat" ) == files[i].length()-4 )
            m_usedFiles.push_back ( files[i] );
    int size = m_usedFiles.size();

    // alloc mem
    m_cascadeInputs = new REAL[size*m_nClass*m_nDomain*m_nTrain];
    for ( int i=0;i<size*m_nClass*m_nDomain*m_nTrain;i++ )
        m_cascadeInputs[i] = 1e10;

    // fill cascadeInputs
    for ( int i=0;i<size;i++ )
    {
        fstream f ( m_usedFiles[i].c_str(), ios::in );
        if ( f.is_open() == false )
            assert ( false );
        REAL* cache = new REAL[m_nTrain*m_nClass*m_nDomain];
        f.read ( ( char* ) cache, sizeof ( REAL ) *m_nTrain*m_nClass*m_nDomain );
        f.close();

        for ( int j=0;j<m_nTrain;j++ )
            for ( int k=0;k<m_nClass*m_nDomain;k++ )
                m_cascadeInputs[j*m_nClass*m_nDomain*size + i*m_nClass*m_nDomain + k] = cache[j*m_nClass*m_nDomain + k];

        if ( cache )
            delete[] cache;
        cache = 0;
    }
    for ( int i=0;i<size;i++ )
    {
        double rmse = 0.0, err;
        for ( int j=0;j<m_nTrain;j++ )
            for ( int k=0;k<m_nClass*m_nDomain;k++ )
            {
                err = m_cascadeInputs[j*m_nClass*m_nDomain*size + i*m_nClass*m_nDomain + k] - m_trainTargetOrig[k + j*m_nClass*m_nDomain];
                rmse += err*err;
            }
        cout<<"File:"<<m_usedFiles[i]<<"  RMSE:"<<sqrt ( rmse/ ( double ) ( m_nClass*m_nTrain*m_nDomain ) ) <<endl;
    }
    if ( size == 0 )
        cout<<"Nothing to do here"<<endl;
    cout<<endl;

    m_nCascadeInputs = size;
    cout<<"nCascadeInputs:"<<m_nCascadeInputs<<endl;
}

void Data::extendTrainDataWithCascadeInputs()
{
    if ( m_nCascadeInputs == 0 )
        return;

    cout<<"Extend the train data with cascade inputs"<<endl;

    if ( m_trainOrig )
    {
        REAL* m_trainOrigNew = new REAL[m_nTrain* ( m_nFeatures+m_nCascadeInputs*m_nClass*m_nDomain ) ];
        for ( int i=0;i<m_nTrain;i++ )
        {
            REAL* ptr0 = m_trainOrigNew + i* ( m_nFeatures+m_nCascadeInputs*m_nClass*m_nDomain );
            REAL* ptr1 = m_trainOrig + i*m_nFeatures;
            for ( int j=0;j<m_nFeatures;j++ )
                ptr0[j] = ptr1[j];
            ptr0 = m_trainOrigNew + i* ( m_nFeatures+m_nCascadeInputs*m_nClass*m_nDomain ) + m_nFeatures;
            ptr1 = m_cascadeInputs + i*m_nCascadeInputs*m_nClass*m_nDomain;
            for ( int j=0;j<m_nCascadeInputs*m_nClass*m_nDomain;j++ )
                ptr0[j] = ptr1[j];
        }
        if ( m_trainOrig )
            delete[] m_trainOrig;
        m_trainOrig = m_trainOrigNew;
    }

    if ( m_testOrig )
    {
        REAL* m_testOrigNew = new REAL[m_nTest* ( m_nFeatures+m_nCascadeInputs*m_nClass*m_nDomain ) ];
        for ( int i=0;i<m_nTest;i++ )
        {
            REAL* ptr0 = m_testOrigNew + i* ( m_nFeatures+m_nCascadeInputs*m_nClass*m_nDomain );
            REAL* ptr1 = m_testOrig + i*m_nFeatures;
            for ( int j=0;j<m_nFeatures;j++ )
                ptr0[j] = ptr1[j];
            ptr0 = m_testOrigNew + i* ( m_nFeatures+m_nCascadeInputs*m_nClass*m_nDomain ) + m_nFeatures;
            ptr1 = m_cascadeInputs + i*m_nCascadeInputs*m_nClass*m_nDomain;
            for ( int j=0;j<m_nCascadeInputs*m_nClass*m_nDomain;j++ )
                ptr0[j] = ptr1[j];
        }
        if ( m_testOrig )
            delete[] m_testOrig;
        m_testOrig = m_testOrigNew;
    }

    int nFeaturesBefore = m_nFeatures;
    m_nFeatures += m_nCascadeInputs*m_nClass*m_nDomain;
    cout<<"nFeatures: "<<m_nFeatures<<" (before: "<<nFeaturesBefore<<")"<<endl;
}


void Data::setPathes ( string temp, string dsc, string fullPred, string data )
{
    m_tempPath = temp;
    m_dscPath = dsc;
    m_fullPredPath = fullPred;
    m_dataPath = data;
}

void Data::readParameter ( string line, int mode )
{
    // split into 2 strings at the '=' char
    int pos = line.find ( "=" );
    string name = line.substr ( 0, pos );
    string value = line.substr ( pos+1 );

    if ( mode==-1 ) // meta info block (algorithm independent)
    {
        if ( name=="ALGORITHM" )
            m_algorithmName = value;
        if ( name=="ID" )
            m_algorithmID = atoi ( value.c_str() );
        if ( name=="TRAIN_ON_FULLPREDICTOR" )
        {
            if(m_validationType == "ValidationSet")
                assert(false);
            m_trainOnFullPredictorFile = value;
        }
        if ( name=="DISABLE" )
            m_disableTraining = atoi ( value.c_str() );
        cout<<"[META] ";
    }

    if ( mode==0 ) // [int]
        m_intMap[name] = atoi ( value.c_str() );

    if ( mode==1 ) // [double]
        m_doubleMap[name] = atof ( value.c_str() );

    if ( mode==2 ) // [string]
        m_stringMap[name] = value;

    if ( mode==3 ) // [bool]
        m_boolMap[name] = atoi ( value.c_str() );

    cout<<name<<": "<<value<<endl;
}

void Data::readDscFile ( string name )
{
    cout<<"Load descriptor file: "<<name<<endl;
    fstream f ( name.c_str(), ios::in );

    if ( f.is_open() ==false )
    {
        cout<<"Can not open file:"<<name<<endl;
        assert ( false );
    }

    int mode = -1;  // -1:meta info  0:int  1:double  2:string  3:bool

    char buf[256];
    while ( f.getline ( buf, 256 ) ) // read all lines
    {
        string line ( buf );
        if ( line[0]=='#' ) // a comment
            continue;
        if ( line.find ( "[int]" ) != string::npos )
            mode = 0;
        if ( line.find ( "[double]" ) != string::npos )
            mode = 1;
        if ( line.find ( "[string]" ) != string::npos )
            mode = 2;
        if ( line.find ( "[bool]" ) != string::npos )
            mode = 3;

        // only lines which consists of a '='
        if ( line.find ( "=" ) != string::npos )
            readParameter ( line, mode );
    }

    f.close();
}

vector<string> Data::getDirectoryFileList ( string path )
{
    vector<string> v;
    DIR *dp;
    struct dirent *dirp;
    if ( ( dp = opendir ( path.c_str() ) ) == NULL )
    {
        cout << "Error opening " << path << endl;
        return v;
    }
    while ( ( dirp = readdir ( dp ) ) != NULL )
        v.push_back ( path + string ( dirp->d_name ) );
    closedir ( dp );
    return v;
}

int* Data::splitStringToIntegerList ( string str, char delimiter )
{
    vector<int> v;
    int number;
    char *begin = ( char* ) str.c_str(), *end = ( char* ) str.c_str(), tmp;
    for ( int i=0;i<str.length();i++ )
    {
        end++;
        if ( *end==delimiter || *end==0 )
        {
            tmp = *end;
            *end = 0;
            sscanf ( begin, "%d", &number );
            begin = end + 1;
            *end = tmp;
            v.push_back ( number );
        }
    }
    int* returnList = new int[v.size() ];
    for ( int i=0;i<v.size();i++ )
        returnList[i] = v[i];
    return returnList;
}

vector<string> Data::splitStringToStringList ( string str, char delimiter )
{
    vector<string> v;
    int number;
    char *begin = ( char* ) str.c_str(), *end = ( char* ) str.c_str(), tmp;
    for ( int i=0;i<str.length();i++ )
    {
        end++;
        if ( *end==delimiter || *end==0 )
        {
            tmp = *end;
            *end = 0;
            v.push_back ( begin );
            begin = end + 1;
            *end = tmp;
        }
    }
    return v;
}

void Data::setDataPointers ( Data* data )
{
    cout<<"Set data pointers"<<endl;

    // copy maps
    m_intMap = data->m_intMap;
    m_doubleMap = data->m_doubleMap;
    m_boolMap = data->m_boolMap;
    m_stringMap = data->m_stringMap;

    m_algorithmName = data->m_algorithmName;
    m_algorithmID = data->m_algorithmID;
    m_trainOnFullPredictorFile = data->m_trainOnFullPredictorFile;
    m_disableTraining = data->m_disableTraining;

    m_randSeed = data->m_randSeed;
    m_positiveTarget = data->m_positiveTarget;
    m_negativeTarget = data->m_negativeTarget;

    m_mixList = data->m_mixList;

    // dataset pathes
    m_datasetPath = data->m_datasetPath;
    m_datasetName = data->m_datasetName;
    m_tempPath = data->m_tempPath;
    m_dscPath = data->m_dscPath;
    m_fullPredPath = data->m_fullPredPath;
    m_dataPath = data->m_dataPath;

    // dataset organization (input/output dimensionality)
    m_nFeatures = data->m_nFeatures;
    m_nClass = data->m_nClass;
    m_nDomain = data->m_nDomain;
    m_nMixTrainList = data->m_nMixTrainList;

    // cross-validation settings
    m_nCross = data->m_nCross;
    m_validationType = data->m_validationType;

    // global mean and standard deviation over whole dataset
    m_mean = data->m_mean;
    m_std = data->m_std;
    m_standardDeviationMin = data->m_standardDeviationMin;
    m_targetMean = data->m_targetMean;

    // full training set
    m_nTrain = data->m_nTrain;
    m_trainOrig = data->m_trainOrig;
    m_trainTargetOrig = data->m_trainTargetOrig;
    m_trainTargetOrigEffect = data->m_trainTargetOrigEffect;
    m_trainTargetOrigResidual = data->m_trainTargetOrigResidual;
    m_trainLabelOrig = data->m_trainLabelOrig;
    m_trainBaggingIndex = data->m_trainBaggingIndex;

    // the validation set
    m_validSize = data->m_validSize;
    m_valid = data->m_valid;
    m_validTarget = data->m_validTarget;
    m_validLabel = data->m_validLabel;
    
    // the testset
    m_nTest = data->m_nTest;
    m_testOrig = data->m_testOrig;
    m_testTargetOrig = data->m_testTargetOrig;
    m_testLabelOrig = data->m_testLabelOrig;

    // probe split inices
    m_slotBoundaries = data->m_slotBoundaries;

    // trainsets per cross-validation division
    m_trainSize = data->m_trainSize;
    m_train = data->m_train;
    m_trainTarget = data->m_trainTarget;
    m_trainTargetEffect = data->m_trainTargetEffect;
    m_trainTargetResidual = data->m_trainTargetResidual;
    m_trainLabel = data->m_trainLabel;

    // probesets per cross-validation division
    m_probeSize = data->m_probeSize;
    m_probe = data->m_probe;
    m_probeTarget = data->m_probeTarget;
    m_probeTargetEffect = data->m_probeTargetEffect;
    m_probeTargetResidual = data->m_probeTargetResidual;
    m_probeLabel = data->m_probeLabel;
    m_probeIndex = data->m_probeIndex;

    m_crossIndex = data->m_crossIndex;

    // blend stopping
    m_blendingRegularization = data->m_blendingRegularization;
    m_enableGlobalBlendingWeights = data->m_enableGlobalBlendingWeights;
    m_blendingEnableCrossValidation = data->m_blendingEnableCrossValidation;
    m_enablePostNNBlending = data->m_enablePostNNBlending;
    m_blendingAlgorithm = data->m_blendingAlgorithm;

    // cascade learning
    m_enableCascadeLearning = data->m_enableCascadeLearning;
    m_nCascadeInputs = data->m_nCascadeInputs;
    m_cascadeInputs = data->m_cascadeInputs;

    // average over mean and std as new mean and std
    m_enableGlobalMeanStdEstimate = data->m_enableGlobalMeanStdEstimate;

    // paralellization of k-fold cross validation
    m_maxThreadsInCross = data->m_maxThreadsInCross;

    // memory save option
    m_enableSaveMemory = data->m_enableSaveMemory;

    // error function "AUC" or "RMSE"
    m_errorFunction = data->m_errorFunction;

    // reverse mix table
    m_mixDatasetIndices = data->m_mixDatasetIndices;

    // already trained algo list
    m_algorithmNameList = data->m_algorithmNameList;

    // clip after blend
    m_enablePostBlendClipping = data->m_enablePostBlendClipping;

    // add output noise
    m_addOutputNoise = data->m_addOutputNoise;

    // feature selection
    m_enableFeatureSelection = data->m_enableFeatureSelection;
    m_featureSelectionWriteBinaryDataset = data->m_featureSelectionWriteBinaryDataset;

    // bagging
    m_enableBagging = data->m_enableBagging;
    m_randomSeedBagging = data->m_randomSeedBagging;

    // write dsc files in training
    m_disableWriteDscFile = data->m_disableWriteDscFile;

    // static mean and std normalization
    m_enableStaticNormalization = data->m_enableStaticNormalization;
    m_staticMeanNormalization = data->m_staticMeanNormalization;
    m_staticStdNormalization = data->m_staticStdNormalization;
    m_enableProbablisticNormalization = data->m_enableProbablisticNormalization;

    // dimensionality reduction
    m_dimensionalityReduction = data->m_dimensionalityReduction;

    // if this is set, the algorithm should load saved weights before start to training
    m_loadWeightsBeforeTraining = data->m_loadWeightsBeforeTraining;

    m_subsampleTrainSet = data->m_subsampleTrainSet;
    m_subsampleFeatures = data->m_subsampleFeatures;
    m_globalTrainingLoops = data->m_globalTrainingLoops;
    m_addConstantInput = data->m_addConstantInput;
}

void Data::setAlgorithmList ( vector<string> algorithmNameList )
{
    cout<<"Set algorithm list (nTrained:"<< ( int ) algorithmNameList.size() <<")"<<endl;
    m_algorithmNameList = algorithmNameList;
    for ( int i=0;i<m_algorithmNameList.size();i++ )
    {
        int pos = m_algorithmNameList[i].find_first_of ( ".",0 );
        if ( pos == 0 )
            assert ( false );
        m_algorithmNameList[i] = m_datasetPath + "/" + m_fullPredPath + "/" + m_algorithmNameList[i].substr ( 0,pos ) + ".dat";
        cout<<"m_algorithmNameList["<<i<<"]:"<<m_algorithmNameList[i]<<endl;
    }
}

void Data::enableBagging ( bool en )
{
    cout<<"Enable bagging:"<<en<<endl;
    m_enableBagging = en;
}

void Data::baggingRandomSeed ( uint seed )
{
    m_randomSeedBagging = seed;
}

void Data::mergeTrainAndTest()
{
    cout<<"trainSet = {trainSet(#"<<m_nTrain<<") + testSet(#"<<m_nTest<<")}"<<endl;
    if ( m_nTest == 0 )
        return;

    REAL* train = new REAL[ ( m_nTrain + m_nTest ) *m_nFeatures];
    REAL* trainTarget = new REAL[ ( m_nTrain + m_nTest ) *m_nClass*m_nDomain];
    int* trainLabel = new int[ ( m_nTrain + m_nTest ) *m_nDomain];

    memcpy ( train, m_trainOrig, sizeof ( REAL ) *m_nTrain*m_nFeatures );
    memcpy ( train + m_nTrain*m_nFeatures, m_testOrig, sizeof ( REAL ) *m_nTest*m_nFeatures );

    memcpy ( trainTarget, m_trainTargetOrig, sizeof ( REAL ) *m_nTrain*m_nClass*m_nDomain );
    memcpy ( trainTarget + m_nTrain*m_nClass*m_nDomain, m_testTargetOrig, sizeof ( REAL ) *m_nTest*m_nClass*m_nDomain );

    memcpy ( trainLabel, m_trainLabelOrig, sizeof ( REAL ) *m_nTrain*m_nDomain );
    memcpy ( trainLabel + m_nTrain*m_nDomain, m_testLabelOrig, sizeof ( REAL ) *m_nTest*m_nDomain );

    delete[] m_trainOrig;
    delete[] m_trainTargetOrig;
    delete[] m_trainLabelOrig;

    m_trainOrig = train;
    m_trainTargetOrig = trainTarget;
    m_trainLabelOrig = trainLabel;

    m_nTrain = m_nTrain + m_nTest;
}

void Data::normalizeZeroOne()
{
    cout<<"Autoencoder: Normalize train between 0 and 1"<<endl;
    // (m_trainOrig[i*m_nFeatures + j] - m_mean[j]) / m_std[j]
    REAL* mean = new REAL[m_nFeatures];
    REAL* std = new REAL[m_nFeatures];

    for ( int i=0;i<m_nFeatures;i++ )
    {
        double mu = 0.0, min = 1e10, max = -1e10;
        for ( int j=0;j<m_nTrain;j++ )
        {
            REAL v = m_trainOrig[i+j*m_nFeatures];
            mu += v;
            if ( min > v )
                min = v;
            if ( max < v )
                max = v;
        }
        mean[i] = min;
        std[i] = max - min;
        if ( std[i] <= 1e-2 )
            std[i] = 1.0;
        m_mean[i] = 0.0;
        m_std[i] = 1.0;

        if ( m_enableStaticNormalization ) // something special, allow to modify the auto normalizations
        {
            mean[i] += m_staticMeanNormalization;
            std[i] *= m_staticStdNormalization;
        }
    }
    for ( int i=0;i<m_nTrain;i++ )
        for ( int j=0;j<m_nFeatures;j++ )
        {
            m_trainOrig[j+i*m_nFeatures] = ( m_trainOrig[j+i*m_nFeatures] - mean[j] ) / std[j];
            REAL v = m_trainOrig[j+i*m_nFeatures];
            if ( v > 1.0 || v < 0.0 )
            {
                cout<<"v:"<<v<<endl;
                assert ( false );
            }
        }

    // print mean/std
    for ( int j=0;j<m_nFeatures;j++ )
        cout<<mean[j]<<"|"<<std[j]<<" ";
    cout<<endl;

    // save the normalizations
    cout<<"save the 0..1 normalizations"<<endl;
    string meanName = m_datasetPath + "/" + m_tempPath + "/AutoencoderDataMean.dat";
    string stdName = m_datasetPath + "/" + m_tempPath + "/AutoencoderDataStd.dat";
    cout<<"meanName:"<<meanName<<endl<<"stdName:"<<stdName<<endl;
    fstream fMean ( meanName.c_str(),ios::out );
    fstream fStd ( stdName.c_str(),ios::out );
    fMean.write ( ( char* ) mean, sizeof ( REAL ) *m_nFeatures );
    fStd.write ( ( char* ) std, sizeof ( REAL ) *m_nFeatures );
    fMean.close();
    fStd.close();

    delete[] mean;
    delete[] std;
}

void Data::reduceTrainingSetSize ( REAL percent )
{
    cout<<"reduce training set (current size:"<<m_nTrain<<") to "<<percent*100.0<<"% of its original size"<<flush;
    if ( percent <= 0.0 || percent >= 1.0 )
    {
        cout<<"  [nothing to do]"<<endl;
        return;
    }
    cout<<endl;
    
    srand ( Framework::getRandomSeed() );
    int cnt = 0;
    for ( int i=0;i<m_nTrain;i++ )
        if ( ( double ) rand() / ( double ) RAND_MAX < percent )
            cnt++;

    cout<<"allocate new training set, size:"<<cnt<<endl;

    REAL* train = new REAL[cnt*m_nFeatures];
    REAL* trainTarget = new REAL[cnt*m_nClass*m_nDomain];

    int* trainLabel = 0;
    if ( m_trainLabelOrig )
        trainLabel = new int[cnt*m_nDomain];

    srand ( Framework::getRandomSeed() );
    cnt = 0;
    for ( int i=0;i<m_nTrain;i++ )
    {
        if ( ( double ) rand() / ( double ) RAND_MAX < percent )
        {
            for ( int j=0;j<m_nFeatures;j++ )
                train[j+cnt*m_nFeatures] = m_trainOrig[j+i*m_nFeatures];
            for ( int j=0;j<m_nClass*m_nDomain;j++ )
                trainTarget[j+cnt*m_nClass*m_nDomain] = m_trainTargetOrig[j+i*m_nClass*m_nDomain];
            if ( m_trainLabelOrig )
            {
                for ( int j=0;j<m_nDomain;j++ )
                    trainLabel[j+cnt*m_nDomain] = m_trainLabelOrig[j+i*m_nDomain];
            }
            cnt++;
        }
    }

    delete[] m_trainOrig;
    delete[] m_trainTargetOrig;
    if ( m_trainLabelOrig )
        delete[] m_trainLabelOrig;

    m_trainOrig = train;
    m_trainTargetOrig = trainTarget;
    if ( m_trainLabelOrig )
        m_trainLabelOrig = trainLabel;

    m_nTrain = cnt;
}

void Data::reduceFeatureSize ( REAL* &table, int tableRows, int &tableCols, REAL percent, bool loadColumnSet )
{
    cout<<"subsample the columns (current:"<<tableCols<<") to "<<percent*100.0<<"% of columns (skip constant 1 features)"<<flush;
    if ( percent <= 0.0 || percent >= 1.0 )
    {
        cout<<"  [nothing to do]"<<endl;
        return;
    }
    cout<<endl;
    
    // determine constant 1 features
    bool* isConstantOne = new bool[tableCols];
    bool* selectedCols = new bool[tableCols];
    for ( int i=0;i<tableCols;i++ )
    {
        isConstantOne[i] = true;
        selectedCols[i] = false;
    }
    for ( int i=0;i<tableRows;i++ )
        for ( int j=0;j<tableCols;j++ )
            isConstantOne[j] &= table[j+i*tableCols]==1.0;

    srand ( Framework::getRandomSeed() );
    int cnt = 0;
    for ( int i=0;i<tableCols;i++ )
        if ( ( double ) rand() / ( double ) RAND_MAX < percent || isConstantOne[i] )
        {
            selectedCols[i] = true;
            cnt++;
        }
    delete[] isConstantOne;

    if ( loadColumnSet )
    {
        string fname = m_datasetPath + "/" + string ( DATA_PATH ) + "/subspace.txt";
        cout<<"load subspace file:"<<fname<<endl;
        fstream f ( fname.c_str(),ios::in );
        cnt = 0;
        for ( int i=0;i<tableCols;i++ )
        {
            f>>selectedCols[i];
            cnt += selectedCols[i];
        }
        f.close();
    }
    else
    {
        string fname = m_datasetPath + "/" + string ( DATA_PATH ) + "/subspace.txt";
        cout<<"write subspace file:"<<fname<<endl;
        fstream f ( fname.c_str(),ios::out );
        for ( int i=0;i<tableCols;i++ )
            f<<selectedCols[i]<<endl;
        f.close();
    }

    cout<<"allocate new table set, column size:"<<cnt<<endl;
    REAL* newTable = new REAL[cnt*tableRows];

    srand ( Framework::getRandomSeed() );
    for ( int i=0;i<tableRows;i++ )
    {
        int c = 0;
        for ( int j=0;j<tableCols;j++ )
        {
            if ( selectedCols[j] )
            {
                newTable[c+i*cnt] = table[j+i*tableCols];
                c++;
            }
        }
    }

    delete[] table;
    delete[] selectedCols;
    table = newTable;
    tableCols = cnt;
}

void Data::addConstantInput()
{
    if(m_trainOrig)
    {
        cout<<"Add a constant 1 column to the train feature matrix"<<endl;
        REAL* trainTmp = new REAL[m_nTrain*(m_nFeatures+1)];
        for(int i=0;i<m_nTrain;i++)
        {
            for(int j=0;j<m_nFeatures;j++)
                trainTmp[i*(m_nFeatures+1)+j] = m_trainOrig[i*m_nFeatures+j];
            trainTmp[i*(m_nFeatures+1)+m_nFeatures] = 1.0;
        }
        delete[] m_trainOrig;
        m_trainOrig = trainTmp;
    }
    if(m_testOrig)
    {
        cout<<"Add a constant 1 column to the test feature matrix"<<endl;
        REAL* testTmp = new REAL[m_nTest*(m_nFeatures+1)];
        for(int i=0;i<m_nTest;i++)
        {
            for(int j=0;j<m_nFeatures;j++)
                testTmp[i*(m_nFeatures+1)+j] = m_testOrig[i*m_nFeatures+j];
            testTmp[i*(m_nFeatures+1)+m_nFeatures] = 1.0;
        }
        delete[] m_testOrig;
        m_testOrig = testTmp;
    }
    m_nFeatures++;
}