Data.h

#ifndef _DATA_H__
#define _DATA_H__

#include <assert.h>
#include <stdio.h>
#include <stdlib.h>
//#include <iostream>
#include <fstream>
#include <string>
#include <math.h>
#include <vector>
#include <map>
#include <dirent.h>
#include <algorithm>

using namespace std;

#include "StreamOutput.h"
#include "DatasetReader.h"
#include "Framework.h"
#include "InputFeatureSelector.h"

class Data : public Framework
{
    // These two classes have full access to all members
    friend class Scheduler;
    friend class Algorithm;
    friend class Autoencoder;

public:
    Data();
    virtual ~Data();

    void readParameter ( string line, int mode );
    void readDscFile ( string name );
    void setPathes ( string temp, string dsc, string fullPred, string data );

    void readDataset ( string name );
    void allocMemForCrossValidationSets();

    void partitionDatasetToCrossValidationSets();
    void fillCascadeLearningInputs();
    void extendTrainDataWithCascadeInputs();
    void fillNCrossValidationSet ( int n );
    void freeNCrossValidationSet ( int n );
    void readEffectFile();

    static vector<string> getDirectoryFileList ( string path );
    static int* splitStringToIntegerList ( string str, char delimiter );
    static vector<string> splitStringToStringList ( string str, char delimiter );

    void setDataPointers ( Data* data );
    void mixDataset();

    void deleteMemory();
    void loadNormalization ( int nCascade = 0 );

    void setAlgorithmList ( vector<string> m_algorithmNameList );

    void loadFeatureSelectionFile();
    void saveFeatureSelectionFile();

    void doFeatureSelection();
    void makeBinaryDataset();

    void enableBagging ( bool en );
    void doBootstrapSampling ( REAL* probs, REAL* &train, REAL* &target, REAL* &targetEff, REAL* &targetRes, int* &label, int nTrainNew = 0 );
    void baggingRandomSeed ( uint seed );

    int vectorSampling ( REAL* probs, int length );

    void mergeTrainAndTest();
    void normalizeZeroOne();

    void reduceTrainingSetSize ( REAL percent );
    void reduceFeatureSize ( REAL* &table, int tableRows, int &tableCols, REAL percent, bool loadColumnSet );

    void addConstantInput();
    
protected:
    // master metadata
    string m_datasetPath;
    string m_datasetName;
    string m_algorithmName;
    int m_algorithmID;
    string m_trainOnFullPredictorFile;
    bool m_disableTraining;
    int m_randSeed;
    int m_nMixDataset;
    int m_nMixTrainList;
    int m_nCross;
    string m_validationType;
    int m_maxThreadsInCross;
    bool m_enableGlobalMeanStdEstimate;
    REAL m_positiveTarget;
    REAL m_negativeTarget;
    double m_blendingRegularization;
    bool m_enableGlobalBlendingWeights;
    bool m_blendingEnableCrossValidation;
    bool m_enablePostNNBlending;
    string m_blendingAlgorithm;

    // enable for cascade learning (where the outcome[targets] of the
    // layer1-algo is added to the input features of the layer2-algo)
    bool m_enableCascadeLearning;
    int m_nCascadeInputs;
    REAL* m_cascadeInputs;

    // read strings from the algorithm's description file
    map<string,int> m_intMap;
    map<string,double> m_doubleMap;
    map<string,bool> m_boolMap;
    map<string,string> m_stringMap;

    // string pathes
    string m_tempPath;
    string m_dscPath;
    string m_fullPredPath;
    string m_dataPath;

    // dataset organization (input/output dimensions)
    int m_nFeatures;
    int m_nClass;
    int m_nDomain;  // number of target classes, 1 is usual

    // randomized train sample list
    int* m_mixDatasetIndices;
    int* m_mixList;
    int* m_crossIndex;

    // full training set
    uint m_nTrain;
    REAL* m_trainOrig;
    REAL* m_trainTargetOrig;
    REAL* m_trainTargetOrigEffect;
    REAL* m_trainTargetOrigResidual;
    int* m_trainLabelOrig;

    // the testset
    uint m_nTest;
    REAL* m_testOrig;
    REAL* m_testTargetOrig;
    int* m_testLabelOrig;

    // probe split bounds
    int* m_slotBoundaries;

    // trainsets per cross-validation division (randomized)
    int* m_trainSize;
    REAL** m_train;
    REAL** m_trainTarget;
    REAL** m_trainTargetEffect;
    REAL** m_trainTargetResidual;
    int** m_trainLabel;
    int** m_trainBaggingIndex;

    // probesets per cross-validation division (randomized)
    int* m_probeSize;
    REAL** m_probe;
    REAL** m_probeTarget;
    REAL** m_probeTargetEffect;
    REAL** m_probeTargetResidual;
    int** m_probeLabel;
    int** m_probeIndex;

    // validationset (fix random subset of train)
    int m_validSize;
    REAL* m_valid;
    REAL* m_validTarget;
    int* m_validLabel;
    
    // global mean and standard deviation over whole dataset
    REAL* m_mean;
    REAL* m_std;
    REAL m_standardDeviationMin;
    REAL* m_targetMean;

    // allocate memory (input feautures) only for active train threads
    bool m_enableSaveMemory;

    // error function
    string m_errorFunction;

    REAL* m_support;

    // algorithm list from scheduler
    vector<string> m_algorithmNameList;

    // clip at the end
    bool m_enablePostBlendClipping;

    // add output noise
    REAL m_addOutputNoise;

    // enable feature selection heuristics
    bool m_enableFeatureSelection;
    bool m_featureSelectionWriteBinaryDataset;

    // enable bagging: done by resampling of the trainingset in retraining
    bool m_enableBagging;
    uint m_randomSeedBagging;

    // write dsc files in training
    bool m_disableWriteDscFile;

    // static mean and std normalization of input features
    bool m_enableStaticNormalization;
    REAL m_staticMeanNormalization;
    REAL m_staticStdNormalization;
    bool m_enableProbablisticNormalization;

    // dimensionality reduction
    string m_dimensionalityReduction;

    // reduce the size of the training set
    REAL m_subsampleTrainSet;

    // reduce the size of the features
    REAL m_subsampleFeatures;

    // global train loops: optimize all algos in the ensemble n times (default value=1)
    int m_globalTrainingLoops;

    // add a constant 1 column to the feature matrix
    bool m_addConstantInput;
    
    // if this is set, the algorithm should load saved weights before start to training
    bool m_loadWeightsBeforeTraining;
};


#endif