DatasetReader Class Reference

#include <DatasetReader.h>

Inheritance diagram for DatasetReader:

List of all members.

Public Member Functions
	DatasetReader ()
	~DatasetReader ()
void	getDataBounds (const char **filenames, string delimiter, int &nFeat, int &nClass, uint &nLines, char *columnType, char *enabledCol, int targetColumn, int filenameID, bool fillData=false, REAL *data=0, int *labels=0, bool addConstantOne=true, bool skipFirstLine=false)
void	splitRandomTestset (REAL percentTest, REAL *data, int *labels, int nData, int nFeat, int nClass, REAL *&train, int *&trainLabel, REAL *&trainTarget, REAL *&test, int *&testLabel, REAL *&testTarget, uint &nTrain, uint &nTest, REAL positiveTarget, REAL negativeTarget, bool noRandom=false)
void	makeNumericTrainAndTestTargets (int nClass, int nTrain, int nTest, REAL positiveTarget, REAL negativeTarget, int *trainLabel, int *testLabel, REAL *&trainTarget, REAL *&testTarget)
void	readMNIST (string path, REAL *&train, REAL *&trainTarget, int *&trainLabel, REAL *&test, REAL *&testTarget, int *&testLabel, uint &nTrain, uint &nTest, int &nClass, int &nDomain, int &nFeat, REAL positiveTarget=1.0, REAL negativeTarget=-1.0)
void	readNETFLIX (string path, REAL *&train, REAL *&trainTarget, int *&trainLabel, REAL *&test, REAL *&testTarget, int *&testLabel, uint &nTrain, uint &nTest, int &nClass, int &nDomain, int &nFeat, REAL positiveTarget=1.0, REAL negativeTarget=-1.0)
void	readKDDCup09Large (string path, REAL *&train, REAL *&trainTarget, int *&trainLabel, REAL *&test, REAL *&testTarget, int *&testLabel, uint &nTrain, uint &nTest, int &nClass, int &nDomain, int &nFeat, REAL positiveTarget=1.0, REAL negativeTarget=-1.0)
void	readKDDCup09LargeBin (string path, REAL *&train, REAL *&trainTarget, int *&trainLabel, REAL *&test, REAL *&testTarget, int *&testLabel, uint &nTrain, uint &nTest, int &nClass, int &nDomain, int &nFeat, REAL positiveTarget=1.0, REAL negativeTarget=-1.0)
void	readKDDCup09Small (string path, REAL *&train, REAL *&trainTarget, int *&trainLabel, REAL *&test, REAL *&testTarget, int *&testLabel, uint &nTrain, uint &nTest, int &nClass, int &nDomain, int &nFeat, REAL positiveTarget=1.0, REAL negativeTarget=-1.0)
void	readAusDM2009 (string path, REAL *&train, REAL *&trainTarget, int *&trainLabel, REAL *&test, REAL *&testTarget, int *&testLabel, uint &nTrain, uint &nTest, int &nClass, int &nDomain, int &nFeat, REAL positiveTarget=1.0, REAL negativeTarget=-1.0)
void	readBINARY (string path, REAL *&train, REAL *&trainTarget, int *&trainLabel, REAL *&test, REAL *&testTarget, int *&testLabel, uint &nTrain, uint &nTest, int &nClass, int &nDomain, int &nFeat, REAL positiveTarget=1.0, REAL negativeTarget=-1.0)
void	readCSV (string path, REAL *&train, REAL *&trainTarget, int *&trainLabel, REAL *&test, REAL *&testTarget, int *&testLabel, uint &nTrain, uint &nTest, int &nClass, int &nDomain, int &nFeat, REAL positiveTarget=1.0, REAL negativeTarget=-1.0)
void	readARFF (string path, REAL *&train, REAL *&trainTarget, int *&trainLabel, REAL *&test, REAL *&testTarget, int *&testLabel, uint &nTrain, uint &nTest, int &nClass, int &nDomain, int &nFeat, REAL positiveTarget=1.0, REAL negativeTarget=-1.0)
void	readPRUDSYS (string path, REAL *&train, REAL *&trainTarget, int *&trainLabel, REAL *&test, REAL *&testTarget, int *&testLabel, uint &nTrain, uint &nTest, int &nClass, int &nDomain, int &nFeat, REAL positiveTarget=1.0, REAL negativeTarget=-1.0)
void	readADULT (string path, REAL *&train, REAL *&trainTarget, int *&trainLabel, REAL *&test, REAL *&testTarget, int *&testLabel, uint &nTrain, uint &nTest, int &nClass, int &nDomain, int &nFeat, REAL positiveTarget=1.0, REAL negativeTarget=-1.0)
void	readAUSTRALIAN (string path, REAL *&train, REAL *&trainTarget, int *&trainLabel, REAL *&test, REAL *&testTarget, int *&testLabel, uint &nTrain, uint &nTest, int &nClass, int &nDomain, int &nFeat, REAL positiveTarget=1.0, REAL negativeTarget=-1.0)
void	readBALANCE (string path, REAL *&train, REAL *&trainTarget, int *&trainLabel, REAL *&test, REAL *&testTarget, int *&testLabel, uint &nTrain, uint &nTest, int &nClass, int &nDomain, int &nFeat, REAL positiveTarget=1.0, REAL negativeTarget=-1.0)
void	readCYLINDERBANDS (string path, REAL *&train, REAL *&trainTarget, int *&trainLabel, REAL *&test, REAL *&testTarget, int *&testLabel, uint &nTrain, uint &nTest, int &nClass, int &nDomain, int &nFeat, REAL positiveTarget=1.0, REAL negativeTarget=-1.0)
void	readBREASTCANCERWISCONSIN (string path, REAL *&train, REAL *&trainTarget, int *&trainLabel, REAL *&test, REAL *&testTarget, int *&testLabel, uint &nTrain, uint &nTest, int &nClass, int &nDomain, int &nFeat, REAL positiveTarget=1.0, REAL negativeTarget=-1.0)
void	readAUSTRALIANCREDIT (string path, REAL *&train, REAL *&trainTarget, int *&trainLabel, REAL *&test, REAL *&testTarget, int *&testLabel, uint &nTrain, uint &nTest, int &nClass, int &nDomain, int &nFeat, REAL positiveTarget=1.0, REAL negativeTarget=-1.0)
void	readDIABETES (string path, REAL *&train, REAL *&trainTarget, int *&trainLabel, REAL *&test, REAL *&testTarget, int *&testLabel, uint &nTrain, uint &nTest, int &nClass, int &nDomain, int &nFeat, REAL positiveTarget=1.0, REAL negativeTarget=-1.0)
void	readGERMAN (string path, REAL *&train, REAL *&trainTarget, int *&trainLabel, REAL *&test, REAL *&testTarget, int *&testLabel, uint &nTrain, uint &nTest, int &nClass, int &nDomain, int &nFeat, REAL positiveTarget=1.0, REAL negativeTarget=-1.0)
void	readGLASS (string path, REAL *&train, REAL *&trainTarget, int *&trainLabel, REAL *&test, REAL *&testTarget, int *&testLabel, uint &nTrain, uint &nTest, int &nClass, int &nDomain, int &nFeat, REAL positiveTarget=1.0, REAL negativeTarget=-1.0)
void	readHEART (string path, REAL *&train, REAL *&trainTarget, int *&trainLabel, REAL *&test, REAL *&testTarget, int *&testLabel, uint &nTrain, uint &nTest, int &nClass, int &nDomain, int &nFeat, REAL positiveTarget=1.0, REAL negativeTarget=-1.0)
void	readHEPATITIS (string path, REAL *&train, REAL *&trainTarget, int *&trainLabel, REAL *&test, REAL *&testTarget, int *&testLabel, uint &nTrain, uint &nTest, int &nClass, int &nDomain, int &nFeat, REAL positiveTarget=1.0, REAL negativeTarget=-1.0)
void	readIONOSPHERE (string path, REAL *&train, REAL *&trainTarget, int *&trainLabel, REAL *&test, REAL *&testTarget, int *&testLabel, uint &nTrain, uint &nTest, int &nClass, int &nDomain, int &nFeat, REAL positiveTarget=1.0, REAL negativeTarget=-1.0)
void	readIRIS (string path, REAL *&train, REAL *&trainTarget, int *&trainLabel, REAL *&test, REAL *&testTarget, int *&testLabel, uint &nTrain, uint &nTest, int &nClass, int &nDomain, int &nFeat, REAL positiveTarget=1.0, REAL negativeTarget=-1.0)
void	readLETTER (string path, REAL *&train, REAL *&trainTarget, int *&trainLabel, REAL *&test, REAL *&testTarget, int *&testLabel, uint &nTrain, uint &nTest, int &nClass, int &nDomain, int &nFeat, REAL positiveTarget=1.0, REAL negativeTarget=-1.0)
void	readMONKS1 (string path, REAL *&train, REAL *&trainTarget, int *&trainLabel, REAL *&test, REAL *&testTarget, int *&testLabel, uint &nTrain, uint &nTest, int &nClass, int &nDomain, int &nFeat, REAL positiveTarget=1.0, REAL negativeTarget=-1.0)
void	readMONKS2 (string path, REAL *&train, REAL *&trainTarget, int *&trainLabel, REAL *&test, REAL *&testTarget, int *&testLabel, uint &nTrain, uint &nTest, int &nClass, int &nDomain, int &nFeat, REAL positiveTarget=1.0, REAL negativeTarget=-1.0)
void	readMONKS3 (string path, REAL *&train, REAL *&trainTarget, int *&trainLabel, REAL *&test, REAL *&testTarget, int *&testLabel, uint &nTrain, uint &nTest, int &nClass, int &nDomain, int &nFeat, REAL positiveTarget=1.0, REAL negativeTarget=-1.0)
void	readMUSHROOM (string path, REAL *&train, REAL *&trainTarget, int *&trainLabel, REAL *&test, REAL *&testTarget, int *&testLabel, uint &nTrain, uint &nTest, int &nClass, int &nDomain, int &nFeat, REAL positiveTarget=1.0, REAL negativeTarget=-1.0)
void	readSATIMAGE (string path, REAL *&train, REAL *&trainTarget, int *&trainLabel, REAL *&test, REAL *&testTarget, int *&testLabel, uint &nTrain, uint &nTest, int &nClass, int &nDomain, int &nFeat, REAL positiveTarget=1.0, REAL negativeTarget=-1.0)
void	readSEGMENTATION (string path, REAL *&train, REAL *&trainTarget, int *&trainLabel, REAL *&test, REAL *&testTarget, int *&testLabel, uint &nTrain, uint &nTest, int &nClass, int &nDomain, int &nFeat, REAL positiveTarget=1.0, REAL negativeTarget=-1.0)
void	readSONAR (string path, REAL *&train, REAL *&trainTarget, int *&trainLabel, REAL *&test, REAL *&testTarget, int *&testLabel, uint &nTrain, uint &nTest, int &nClass, int &nDomain, int &nFeat, REAL positiveTarget=1.0, REAL negativeTarget=-1.0)
void	readVEHICLE (string path, REAL *&train, REAL *&trainTarget, int *&trainLabel, REAL *&test, REAL *&testTarget, int *&testLabel, uint &nTrain, uint &nTest, int &nClass, int &nDomain, int &nFeat, REAL positiveTarget=1.0, REAL negativeTarget=-1.0)
void	readVOTES (string path, REAL *&train, REAL *&trainTarget, int *&trainLabel, REAL *&test, REAL *&testTarget, int *&testLabel, uint &nTrain, uint &nTest, int &nClass, int &nDomain, int &nFeat, REAL positiveTarget=1.0, REAL negativeTarget=-1.0)
void	readWINE (string path, REAL *&train, REAL *&trainTarget, int *&trainLabel, REAL *&test, REAL *&testTarget, int *&testLabel, uint &nTrain, uint &nTest, int &nClass, int &nDomain, int &nFeat, REAL positiveTarget=1.0, REAL negativeTarget=-1.0)
void	readPOKER (string path, REAL *&train, REAL *&trainTarget, int *&trainLabel, REAL *&test, REAL *&testTarget, int *&testLabel, uint &nTrain, uint &nTest, int &nClass, int &nDomain, int &nFeat, REAL positiveTarget=1.0, REAL negativeTarget=-1.0)
void	readYEAST (string path, REAL *&train, REAL *&trainTarget, int *&trainLabel, REAL *&test, REAL *&testTarget, int *&testLabel, uint &nTrain, uint &nTest, int &nClass, int &nDomain, int &nFeat, REAL positiveTarget=1.0, REAL negativeTarget=-1.0)
void	readSURVIVAL (string path, REAL *&train, REAL *&trainTarget, int *&trainLabel, REAL *&test, REAL *&testTarget, int *&testLabel, uint &nTrain, uint &nTest, int &nClass, int &nDomain, int &nFeat, REAL positiveTarget=1.0, REAL negativeTarget=-1.0)
void	readSPIDER (string path, REAL *&train, REAL *&trainTarget, int *&trainLabel, REAL *&test, REAL *&testTarget, int *&testLabel, uint &nTrain, uint &nTest, int &nClass, int &nDomain, int &nFeat, REAL positiveTarget=1.0, REAL negativeTarget=-1.0)

---

Detailed Description

Reads predefined datasets The dataset must have: train + testset For both sets: input features + targets (+ for classification:labels)

Definition at line 30 of file DatasetReader.h.

---

Constructor & Destructor Documentation

DatasetReader::DatasetReader

(

)

Constructor

Definition at line 8 of file DatasetReader.cpp.

{
    cout<<"DatasetReader"<<endl;
}

DatasetReader::~DatasetReader

(

)

Destructor

Definition at line 16 of file DatasetReader.cpp.

{
    cout<<"descructor DatasetReader"<<endl;
}

---

Member Function Documentation

void DatasetReader::getDataBounds	(	const char **	filenames,
		string	delimiter,
		int &	nFeat,
		int &	nClass,
		uint &	nLines,
		char *	columnType,
		char *	enabledCol,
		int	targetColumn,
		int	filenameID,
		bool	fillData = false,
		REAL *	data = 0,
		int *	labels = 0,
		bool	addConstantOne = true,
		bool	skipFirstLine = false
	)

Read from a standard data matrix If numerical values are undefined, assign the mean value

e.g.: (last column is target class) 19910108,X126,NO,LINE,YES,Motter94,1911,55,46,0.2,17,78,0.75,20,13.1,1700,50.5,36.4,0,0,2.5,1,34,40,105,100,band 19910109,X266,NO,LINE,YES,Motter94,?,55,46,0.3,15,80,0.75,20,6.6,1900,54.9,38.5,0,0,2.5,0.7,34,40,105,100,noband 19910104,B7,NO,LINE,YES,WoodHoe70,?,62,40,0.433,16,80,?,30,6.5,1850,53.8,39.8,0,0,2.8,0.9,40,40,103.87,100,noband 19910104,T133,NO,LINE,YES,WoodHoe70,1910,52,40,0.3,16,75,0.3125,30,5.6,1467,55.6,38.8,0,0,2.5,1.3,40,40,108.06,100,noband 19910111,J34,NO,LINE,YES,WoodHoe70,1910,50,46,0.3,17,80,0.75,30,0,2100,57.5,42.5,5,0,2.3,0.6,35,40,106.67,100,noband

Parameters:

	filenames	The dataset names
	delemiter	The delimiter string, e.g.: ", " or ","
	nFeat	Reference to the number of features (output value)
	nLines	Reference to the number of lines in the dataset
	columnType	A char* that select the data type: 'd' for discrete string value, 'n' for numeric value
	enabledCol	A chat* that has '0' or '1', for reject or select a data column
	targetColumn	The number of the column, that holds target classes (begin with 0)
	filenameID	Select the filename in filenames
	fillData	If true: fill REAL* data and int* labels with data
	data	Pointer to data (allocated here)
	labels	Pointer to labels (allocated here)

Definition at line 4510 of file DatasetReader.cpp.

{
    int bufSize = 1024*1024;
    int nFiles = 0;
    while ( filenames[nFiles] )
        nFiles++;
    cout<<"nFiles:"<<nFiles<<endl;

    fstream f;

    for ( int i=0;i<nFiles;i++ )
    {
        f.open ( filenames[i], ios::in );
        if ( f.is_open() == false )
        {
            cout<<"Can not open "<<filenames[i]<<endl;
            exit ( 0 );
        }
        f.close();
    }

    int columnTypeSize = 0;
    while ( columnType[columnTypeSize] )
        columnTypeSize++;
    cout<<"columnTypeSize:"<<columnTypeSize<<endl;
    char buf0[bufSize], buf1[bufSize];
    int delimiterLength = delimiter.length();
    const char* delimiterCharPtr = delimiter.c_str();
    vector<string>* discreteValues = new vector<string>[columnTypeSize];
    double* numericMean = new double[columnTypeSize];
    int* numericMeanCnt = new int[columnTypeSize];
    for ( int i=0;i<columnTypeSize;i++ )
    {
        numericMean[i] = 0.0;
        numericMeanCnt[i] = 0;
    }
    for ( int fileCnt=0;fileCnt<nFiles;fileCnt++ )
    {
        f.open ( filenames[fileCnt], ios::in );
        if ( fileCnt == filenameID )
            nLines = 0;

        if ( skipFirstLine )
            f.getline ( buf0, bufSize );

        while ( f.getline ( buf0, bufSize ) ) // read all lines
        {
            int cnt0 = 0, cnt1 = 0, cellCnt = 0;
            while ( buf0[cnt1] != 0 && cnt1 < bufSize ) // read all chars per line
            {
                int matchCnt = 0;
                for ( int i=0;i<delimiterLength;i++ )
                    matchCnt += delimiterCharPtr[i] == buf0[cnt1+i];

                if ( buf0[cnt1+delimiterLength]!=' ' && cnt1 > 0 && ( matchCnt == delimiterLength || buf0[cnt1+1] == 0 || buf0[cnt1+1] == '\r' ) ) // a delimiter match is found, or end of line
                {
                    if ( cellCnt >= columnTypeSize )
                        break;

                    int addOne = 0;
                    if ( buf0[cnt1+1] == 0 || buf0[cnt1+1] == '\r' )
                        addOne = 1;
                    strncpy ( buf1, buf0 + cnt0, cnt1 - cnt0 + addOne );
                    buf1[cnt1 - cnt0 + addOne] = 0;
                    cnt0 = cnt1 + delimiterLength;
                    if ( cnt1 < cnt0 - 1 )
                        cnt1 = cnt0 - 1;
                    if ( enabledCol[cellCnt] == '1' )
                    {
                        if ( columnType[cellCnt] == 'd' )
                        {
                            // search for existing
                            bool exists = false;
                            for ( int i=0;i<discreteValues[cellCnt].size();i++ )
                                if ( discreteValues[cellCnt][i] == string ( buf1 ) )
                                    exists = true;
                            if ( exists == false )
                                discreteValues[cellCnt].push_back ( string ( buf1 ) );
                        }
                        else if ( columnType[cellCnt] == 'n' )
                        {
                            if ( ( buf1[0] >= '0' && buf1[0] <= '9' ) || buf1[0] == '.' || buf1[0] == '-' ) // is a numeric value
                            {
                                float num;
                                sscanf ( buf1,"%f",&num );
                                if ( fileCnt == filenameID )
                                {
                                    numericMean[cellCnt] += num;
                                    numericMeanCnt[cellCnt]++;
                                }
                            }
                            else  // is an unknown numeric value
                            {
                                ;
                            }
                        }
                        else
                            assert ( false );
                    }
                    //cout<<cellCnt<<":"<<string(buf1)<<"|";
                    cellCnt++;
                    if ( buf0[cnt1+1] == 0 )
                        break;
                }
                else if ( cnt1 == 0 && ( matchCnt == delimiterLength || buf0[cnt1+1] == 0 ) )
                    cnt0++;
                cnt1++;
            }
            //cout<<endl;

            // if the line has content
            if ( cnt1 > 1 )
            {
                if ( cellCnt != columnTypeSize && cellCnt > 1 )
                {
                    cout<<"cellCnt:"<<cellCnt<<" columnTypeSize:"<<columnTypeSize<<endl;
                    assert ( false );
                }
                if ( fileCnt == filenameID )
                    nLines++;
            }
            memset ( buf0, 0, bufSize );
        }
        f.close();

    }

    // calculate the total number of features
    nFeat = 0;
    cout<<"ValuesPerDiscreteInput:"<<endl;
    for ( int i=0;i<columnTypeSize;i++ )
    {
        if ( i+1 != targetColumn )
        {
            if ( enabledCol[i] == '1' )
            {
                if ( columnType[i] == 'd' )
                {
                    cout<<i<<": #"<< ( int ) discreteValues[i].size() <<" {";
                    for ( int j=0;j<discreteValues[i].size();j++ )
                        cout<<discreteValues[i][j]<<",";
                    cout<<"}"<<endl;
                    nFeat += discreteValues[i].size();
                }
                else if ( columnType[i] == 'n' )
                    nFeat++;
                else
                    assert ( false );
            }
        }
    }
    if ( addConstantOne )
        nFeat++;
    cout<<endl;

    nClass = discreteValues[targetColumn-1].size();
    cout<<"#Targets:"<< ( int ) nClass<<" {";
    for ( int j=0;j<nClass;j++ )
    {
        string value = discreteValues[targetColumn-1][j];
        cout<<value<<","<<flush;
    }
    cout<<"}"<<endl;

    cout<<endl;
    cout<<"nFeat:"<<nFeat<<endl;
    cout<<"nLines:"<<nLines<<endl;

    if ( fillData )
    {
        // clear data
        for ( int i=0;i<nLines*nFeat;i++ )
            data[i] = 0.0;
        if ( addConstantOne )
        {
            for ( int i=0;i<nLines;i++ )
                data[i*nFeat + nFeat-1] = 1.0;
        }
        for ( int i=0;i<nLines;i++ )
            labels[i] = 0;

        f.open ( filenames[filenameID], ios::in );
        nLines = 0;

        if ( skipFirstLine )
            f.getline ( buf0, bufSize );

        while ( f.getline ( buf0, bufSize ) ) // read all lines
        {
            int cnt0 = 0, cnt1 = 0, cellCnt = 0, pos = 0;
            while ( buf0[cnt1] != 0 && cnt1 < bufSize ) // read all chars per line
            {
                int matchCnt = 0;
                for ( int i=0;i<delimiterLength;i++ )
                    matchCnt += delimiterCharPtr[i] == buf0[cnt1+i];

                if ( buf0[cnt1+delimiterLength]!=' ' && cnt1 > 0 && ( matchCnt == delimiterLength || buf0[cnt1+1] == 0 || buf0[cnt1+1] == '\r' ) ) // a delimiter match is found, or end of line
                {
                    if ( cellCnt >= columnTypeSize )
                        break;

                    int addOne = 0;
                    if ( buf0[cnt1+1] == 0 || buf0[cnt1+1] == '\r' )
                        addOne = 1;
                    strncpy ( buf1, buf0 + cnt0, cnt1 - cnt0 + addOne );
                    buf1[cnt1 - cnt0 + addOne] = 0;
                    cnt0 = cnt1 + delimiterLength;
                    if ( cnt1 < cnt0 - 1 )
                        cnt1 = cnt0 - 1;
                    if ( enabledCol[cellCnt] == '1' )
                    {
                        if ( columnType[cellCnt] == 'd' ) // discete value: {"Hugo","Bart","Moe",..}
                        {
                            // search in existing values
                            int searchPos = -1;
                            for ( int i=0;i<discreteValues[cellCnt].size();i++ )
                                if ( discreteValues[cellCnt][i] == string ( buf1 ) )
                                    searchPos = i;

                            if ( searchPos == -1 )
                                assert ( false );

                            // assign value
                            if ( cellCnt+1 == targetColumn )
                            {
                                labels[nLines] = searchPos;
                            }
                            else
                            {
                                data[nLines*nFeat + pos + searchPos] = 1.0;
                                pos += discreteValues[cellCnt].size();
                            }
                        }
                        else if ( columnType[cellCnt] == 'n' ) // numeric value like: 1.23 or .34 or 1.2e3
                        {
                            if ( ( buf1[0] >= '0' && buf1[0] <= '9' ) || buf1[0] == '.' || buf1[0] == '-' ) // is a numeric value
                            {
                                float num;
                                sscanf ( buf1,"%f",&num );
                                data[nLines*nFeat + pos] = num;
                            }
                            else  // is an unknown numeric value
                            {
                                data[nLines*nFeat + pos] = 0.0;
                                if ( numericMeanCnt[cellCnt] > 0 )
                                    data[nLines*nFeat + pos] = numericMean[cellCnt] / numericMeanCnt[cellCnt];
                            }
                            pos++;
                        }
                        else
                            assert ( false );
                    }
                    cellCnt++;
                }
                else if ( cnt1 == 0 && ( matchCnt == delimiterLength || buf0[cnt1+1] == 0 ) )
                    cnt0++;
                cnt1++;
            }

            // if the line has content
            if ( cnt1 > 1 )
            {
                if ( cellCnt != columnTypeSize && cellCnt > 1 )
                {
                    cout<<"cellCnt:"<<cellCnt<<" columnTypeSize:"<<columnTypeSize<<endl;
                    assert ( false );
                }
                nLines++;

                if ( pos != nFeat - ( int ) addConstantOne )
                {
                    cout<<"pos:"<<pos<<" nFeat:"<<nFeat<<endl;
                    assert ( false );
                }
            }
            memset ( buf0, 0, bufSize );
        }
        f.close();

        // check for NANs or INFs or too large numbers
        for ( int i=0;i<nLines*nFeat;i++ )
            if ( isnan ( data[i] ) || isinf ( data[i] ) || data[i]>1e10 || data[i]<-1e10 )
            {
                cout<<"data["<<i<<"]:"<<data[i]<<endl;
                assert ( false );
            }
        for ( int i=0;i<nLines;i++ )
            if ( isnan ( labels[i] ) || isinf ( labels[i] ) || labels[i]<0 )
            {
                cout<<"labels["<<i<<"]:"<<labels[i]<<endl;
                assert ( false );
            }

    }

}

void DatasetReader::makeNumericTrainAndTestTargets	(	int	nClass,
		int	nTrain,
		int	nTest,
		REAL	positiveTarget,
		REAL	negativeTarget,
		int *	trainLabel,
		int *	testLabel,
		REAL *&	trainTarget,
		REAL *&	testTarget
	)

make numeric train and test target vectors

Definition at line 4889 of file DatasetReader.cpp.

{
    // train targets
    trainTarget = new REAL[nClass*nTrain];
    for ( int i=0;i<nTrain;i++ )
    {
        for ( int j=0;j<nClass;j++ )
            trainTarget[i*nClass + j] = negativeTarget;  // negative class labels
        trainTarget[i*nClass + trainLabel[i]] = positiveTarget;  // positive class label
    }

    // test targets
    testTarget = new REAL[nClass*nTest];
    for ( int i=0;i<nTest;i++ )
    {
        for ( int j=0;j<nClass;j++ )
            testTarget[i*nClass + j] = negativeTarget;  // negative class labels
        testTarget[i*nClass + testLabel[i]] = positiveTarget;  // positive class label
    }

    // check for NANs or INFs or too large numbers
    for ( int i=0;i<nTrain*nClass;i++ )
        if ( isnan ( trainTarget[i] ) || isinf ( trainTarget[i] ) || trainTarget[i]>1e10 || trainTarget[i]<-1e10 )
        {
            cout<<"trainTarget["<<i<<"]:"<<trainTarget[i]<<endl;
            assert ( false );
        }

    for ( int i=0;i<nTest*nClass;i++ )
        if ( isnan ( testTarget[i] ) || isinf ( testTarget[i] ) || testTarget[i]>1e10 || testTarget[i]<-1e10 )
        {
            cout<<"testTarget["<<i<<"]:"<<testTarget[i]<<endl;
            assert ( false );
        }

}

void DatasetReader::readADULT	(	string	path,
		REAL *&	train,
		REAL *&	trainTarget,
		int *&	trainLabel,
		REAL *&	test,
		REAL *&	testTarget,
		int *&	testLabel,
		uint &	nTrain,
		uint &	nTest,
		int &	nClass,
		int &	nDomain,
		int &	nFeat,
		REAL	positiveTarget = 1.0,
		REAL	negativeTarget = -1.0
	)

Reads the ADULT dataset (UCI) 3974305Bytes adult.data 5229Bytes adult.names 2003153Bytes adult.test

Definition at line 3266 of file DatasetReader.cpp.

{
    cout<<"Read ADULT from: "<<path<<endl;
    nDomain = 1;

    // define data type and files
    int targetColumn = 15;
    char columnType[] = "ndndndddddnnndd";
    char enabledCol[] = "111111111111111";
    const char* dataFiles[] = { ( new string ( path+"/adult.data" ) )->c_str(), ( new string ( path+"/adult.test" ) )->c_str(),0};

    // === TRAIN SET ===
    getDataBounds ( dataFiles, ", ", nFeat, nClass, nTrain, columnType, enabledCol, targetColumn, 0 );

    // allocate tmp mem
    train = new REAL[nTrain * nFeat];
    trainLabel = new int[nTrain];

    // fill data
    getDataBounds ( dataFiles, ", ", nFeat, nClass, nTrain, columnType, enabledCol, targetColumn, 0, true, train, trainLabel );


    // === TEST SET ===
    getDataBounds ( dataFiles, ", ", nFeat, nClass, nTest, columnType, enabledCol, targetColumn, 1 );

    // allocate tmp mem
    test = new REAL[nTest * nFeat];
    testLabel = new int[nTest];

    // fill data
    getDataBounds ( dataFiles, ", ", nFeat, nClass, nTest, columnType, enabledCol, targetColumn, 1, true, test, testLabel );

    // make numerical targets
    makeNumericTrainAndTestTargets ( nClass, nTrain, nTest, positiveTarget, negativeTarget, trainLabel, testLabel, trainTarget, testTarget );
}

void DatasetReader::readARFF	(	string	path,
		REAL *&	train,
		REAL *&	trainTarget,
		int *&	trainLabel,
		REAL *&	test,
		REAL *&	testTarget,
		int *&	testLabel,
		uint &	nTrain,
		uint &	nTest,
		int &	nClass,
		int &	nDomain,
		int &	nFeat,
		REAL	positiveTarget = 1.0,
		REAL	negativeTarget = -1.0
	)

Read the data in ARFF format see: http://www.cs.waikato.ac.nz/~ml/weka/arff.html

Definition at line 2083 of file DatasetReader.cpp.

{
    cout<<"Read ARFF from: "<<path<<endl;
    nDomain = 1;
    
    char* buf = new char[1024*1024];
    char del = 0;
    string trainName, trainTargetColumn;
    
    // read the settings file
    fstream fSetting(string(path+"/settings.txt").c_str(),ios::in);
    while ( fSetting.getline ( buf, 1024*1024 ) )
    {
        string s = buf;
        size_t pos = s.find_first_of('=');
        string token = s.substr(0,pos);
        cout<<token<<endl;
        if(token == "trainTargetColumn")
            trainTargetColumn = s.substr(pos+1);
        else if(token == "train")
            trainName = s.substr(pos+1);
    }
    fSetting.close();
    
    if(trainName=="" || trainTargetColumn=="")
        assert(false);
    
    // read training set
    fstream fTrain(string(path+"/"+trainName).c_str(),ios::in);
    vector<vector<REAL> > targets;
    vector<vector<REAL> > features;
    vector<string> featureNames;
    vector<map<string,int> > featureValues;
    bool dataMode = false;
    while ( fTrain.getline ( buf, 1024*1024 ) )
    {
        string s = buf;  // the line
        
        if(s.length() == 0)  // no empty lines
            continue;
        if(s[0] == '%')  // skip comments
            continue;
        if(s[0] == '@')  // control sign
        {
            dataMode = false;
            size_t spacePos0 = s.find_first_of(' ');
            string token = s.substr(0,spacePos0);  // token from beginning
            
            if(token == "@relation" || token == "@RELATION")
                cout<<"Dataset name:"<<s.substr(spacePos0+1)<<endl;
            else if(token == "@attribute" || token == "@ATTRIBUTE")
            {
                // @attribute 'family' {'?','GB','GK','GS','TN','ZA','ZF','ZH','ZM','ZS'}
                size_t spacePos1 = s.find_first_of(" \t", spacePos0+1);
                string featureName = s.substr(spacePos0+1,spacePos1-spacePos0-1);
                featureNames.push_back(featureName);
                
                map<string,int> values;
                size_t curlyPos0 = s.find_first_of('{', spacePos1+1);
                size_t curlyPos1 = s.find_first_of('}', spacePos1+1);
                size_t pos = curlyPos0+1;
                if(curlyPos0 != string::npos && curlyPos1 != string::npos)
                {
                    while(pos < s.length())
                    {
                        size_t delPos = s.find_first_of(',',pos);
                        if(delPos==string::npos)
                            delPos = curlyPos1;
                        string feature = s.substr(pos,delPos-pos);
                        while(*(feature.begin()) == ' ')  // remove leading spaces
                            feature = feature.substr(1);
                        if(feature.length() > 0)
                            while(feature[feature.length()-1] == ' ')  // remove ending spaces
                            {
                                feature = feature.substr(0,feature.length()-1);
                                if(feature.length() == 0)
                                    break;
                            }
                        if(feature.length() > 0)
                            values[feature] = values.size();  // assign new id
                        pos += feature.length()+1;
                    }
                }
                featureValues.push_back(values);  // push empty map when having a "real" attribute
            }
            else if(token == "@data" || token == "@DATA")
                dataMode = true;
        }
        else if(dataMode)
        {
            // '?','C','A',8,0,'?','S','?',0,'?','?','G','?','?','?','?','?','?','?','?','?','?','?','?','?','?','?','?','?','?','?','COIL',0.7,610,0,'?','0','?','3'
            //cout<<featureValues.size()<<endl;
            size_t pos = 0;
            uint valueCnt = 0;
            vector<REAL> feature;
            while(pos < s.length())
            {
                size_t delPos = s.find_first_of(',',pos);
                if(delPos==string::npos)
                    delPos = s.length();
                string value = s.substr(pos,delPos-pos);
                //cout<<value<<" "<<featureNames[valueCnt]<<endl;
                if(featureValues[valueCnt].size() == 0)  // check for real-valued attribute
                {
                    if(featureNames[valueCnt] == trainTargetColumn)
                    {
                        vector<REAL> target;
                        target.push_back(atof(value.c_str()));
                        targets.push_back(target);
                    }
                    else
                        feature.push_back(atof(value.c_str()));
                }
                else  // categorical type
                {
                    uint catSize = featureValues[valueCnt].size();
                    if(featureNames[valueCnt] == trainTargetColumn)
                    {
                        vector<REAL> target;
                        map<string,int>::iterator it = featureValues[valueCnt].find(value);
                        for(int i=0;i<catSize;i++)
                            target.push_back(negativeTarget);
                        uint catPos = it->second;
                        target[catPos] = positiveTarget;
                        targets.push_back(target);
                    }
                    else
                    {
                        map<string,int>::iterator it = featureValues[valueCnt].find(value);
                        if(it == featureValues[valueCnt].end())
                            assert(false);
                        for(int i=0;i<catSize;i++)
                            feature.push_back(-1.0);  // init with negative
                        uint catPos = it->second;
                        feature[feature.size()-catSize+catPos] = 1.0;
                    }
                }
                valueCnt++;
                pos += value.length()+1;
            }
            features.push_back(feature);
        }
    }
    fTrain.close();
    
    assert(features.size() == targets.size());
    
    // print a short summary
    nTrain = features.size();
    nFeat = features[0].size();
    nClass = targets[0].size();
    cout<<"nTrain:"<<nTrain<<" nFeat:"<<nFeat<<" nClass:"<<nClass<<" nFeatureNames:"<<featureNames.size()<<endl;
    for(int i=0;i<featureNames.size();i++)
    {
        cout<<"name:"<<featureNames[i]<<" ";
        if(featureValues[i].size() == 0)
            cout<<"[REAL]";
        else
        {
            for(map<string,int>::iterator it = featureValues[i].begin();it != featureValues[i].end(); it++)
                cout<<"\""<<it->first<<"\" ";
        }
        cout<<endl;
    }
    
    // allocate + fill train data
    train = new REAL[nFeat*nTrain];
    trainTarget = new REAL[nClass*nTrain];
    trainLabel = nClass > 1 ? new int[nTrain] : 0;
    
    for(int i=0;i<nTrain;i++)
    {
        for(int j=0;j<nFeat;j++)
            train[i*nFeat+j] = features[i][j];
        for(int j=0;j<nClass;j++)
        {
            trainTarget[i*nClass+j] = targets[i][j];
            if(targets[i][j] == positiveTarget && nClass > 1)
                trainLabel[i] = j;
        }
    }
    
    // no test set
    test = 0;
    testTarget = 0;
    testLabel = 0;
    nTest = 0;
    
    delete[] buf;
}

void DatasetReader::readAusDM2009	(	string	path,
		REAL *&	train,
		REAL *&	trainTarget,
		int *&	trainLabel,
		REAL *&	test,
		REAL *&	testTarget,
		int *&	testLabel,
		uint &	nTrain,
		uint &	nTest,
		int &	nClass,
		int &	nDomain,
		int &	nFeat,
		REAL	positiveTarget = 1.0,
		REAL	negativeTarget = -1.0
	)

AusDM2009 competition http://www.tiberius.biz/ausdm09/ -rw-r--r-- 1 15136906 Sep 25 20:18 S_AUC_Score.csv -rw-r--r-- 1 15131946 Sep 25 20:18 S_AUC_Train.csv -rw-r--r-- 1 15137106 Sep 25 20:18 S_RMSE_Score.csv -rw-r--r-- 1 15171000 Sep 25 20:18 S_RMSE_Train.csv

Definition at line 2510 of file DatasetReader.cpp.

{
    cout<<"Read AusDM2009 from: "<<path<<endl;
    //string nameTrain = "S_RMSE_Train.csv";
    //string nameTest = "S_RMSE_Score.csv";
    //string nameTrain = "M_RMSE_Train.csv";
    //string nameTest = "M_RMSE_Score.csv";
    string nameTrain = "L_RMSE_Train.csv";
    string nameTest = "L_RMSE_Score.csv";
    nClass = 1;
    bool addConstantOne = true;

    if ( Framework::getDatasetType() == 1 ) // is classification
    {
        //nameTrain = "S_AUC_Train.csv";
        //nameTest = "S_AUC_Score.csv";
        //nameTrain = "M_AUC_Train.csv";
        //nameTest = "M_AUC_Score.csv";
        nameTrain = "L_AUC_Train.csv";
        nameTest = "L_AUC_Score.csv";
        nClass = 2;
    }

    cout<<"nameTrain:"<<nameTrain<<" nameTest:"<<nameTest<<endl;

    int bufSize = 1024*1024;
    char *buf = new char[bufSize];

    nDomain = 1;

    fstream fTrainRMSE;
    fstream fTestRMSE;

    // determine #cols
    fTrainRMSE.open ( ( path+"/"+nameTrain ).c_str(), ios::in );
    fTrainRMSE.getline ( buf, bufSize );
    fTrainRMSE.getline ( buf, bufSize );
    nFeat = 0;
    char *ptr = buf, *ptrLast = buf;
    int pos = 0, val, colCnt = 0;
    while ( ptr[pos] )
    {
        if ( ptr[pos] == ',' || ptr[pos+1] == 0 )
        {
            sscanf ( ptrLast,"%d",&val );
            ptrLast = ptr + pos + 1;
            colCnt++;
            if ( colCnt > 2 )
                nFeat++;
        }
        pos++;
    }
    fTrainRMSE.close();

    if ( addConstantOne )
        nFeat++;  // constant one

    // determine #rows train
    fTrainRMSE.open ( ( path+"/"+nameTrain ).c_str(), ios::in );
    fTrainRMSE.getline ( buf, bufSize );
    nTrain = 0;
    while ( fTrainRMSE.getline ( buf, bufSize ) )
        nTrain++;
    fTrainRMSE.close();

    // determine #rows test
    fTestRMSE.open ( ( path+"/"+nameTest ).c_str(), ios::in );
    fTestRMSE.getline ( buf, bufSize );
    nTest = 0;
    while ( fTestRMSE.getline ( buf, bufSize ) )
        nTest++;
    fTestRMSE.close();

    // alloc mem
    train = new REAL[nFeat*nTrain];
    test = new REAL[nFeat*nTest];
    if ( Framework::getDatasetType() == 1 )
    {
        trainTarget = new REAL[nTrain*2];
        trainLabel = new int[nTrain];
        testTarget = new REAL[nTest*2];
        testLabel = new int[nTest];
    }
    else
    {
        trainTarget = new REAL[nTrain];
        trainLabel = 0;
        testTarget = new REAL[nTest];
        testLabel = 0;
    }

    // read train
    fTrainRMSE.open ( ( path+"/"+nameTrain ).c_str(), ios::in );
    fTrainRMSE.getline ( buf, bufSize );
    nTrain = 0;
    while ( fTrainRMSE.getline ( buf, bufSize ) )
    {
        ptr = buf;
        ptrLast = buf;
        pos = 0;
        colCnt = 0;
        while ( ptr[pos] )
        {
            if ( ptr[pos] == ',' || ptr[pos+1] == 0 )
            {
                sscanf ( ptrLast,"%d",&val );
                ptrLast = ptr + pos + 1;
                colCnt++;
                if ( colCnt == 2 )
                {
                    if ( Framework::getDatasetType() == 1 )
                    {
                        trainLabel[nTrain] = val>0? 0 : 1;
                        trainTarget[2*nTrain+0] = val>0? positiveTarget : negativeTarget;
                        trainTarget[2*nTrain+1] = val>0? negativeTarget : positiveTarget;
                    }
                    else
                        trainTarget[nTrain] = ( REAL ) val * 0.001;
                    //trainTarget[nTrain] = (REAL)val;
                }
                if ( colCnt > 2 )
                    train[nTrain*nFeat+colCnt-3] = ( REAL ) val * 0.001;
                //train[nTrain*nFeat+colCnt-3] = (REAL)val;
            }
            pos++;
        }
        if ( ( colCnt-3 != nFeat-1 && addConstantOne == false ) || ( colCnt-3 != nFeat-2 && addConstantOne == true ) )
        {
            cout<<"colCnt:"<<colCnt<<" nFeat:"<<nFeat<<" addConstantOne:"<<addConstantOne<<endl;
            assert ( false );
        }
        if ( addConstantOne )
            train[nTrain*nFeat+nFeat-1] = 1.0;
        nTrain++;
    }
    fTrainRMSE.close();

    // read test
    fTestRMSE.open ( ( path+"/"+nameTest ).c_str(), ios::in );
    fTestRMSE.getline ( buf, bufSize );
    nTest = 0;
    while ( fTestRMSE.getline ( buf, bufSize ) )
    {
        ptr = buf;
        ptrLast = buf;
        pos = 0;
        colCnt = 0;
        while ( ptr[pos] )
        {
            if ( ptr[pos] == ',' || ptr[pos+1] == 0 )
            {
                sscanf ( ptrLast,"%d",&val );
                ptrLast = ptr + pos + 1;
                colCnt++;

                if ( Framework::getDatasetType() == 1 )
                {
                    testTarget[nTest] = val>0? 0 : 1;
                    testTarget[2*nTest+0] = val>0? positiveTarget : negativeTarget;
                    testTarget[2*nTest+1] = val>0? negativeTarget : positiveTarget;
                }
                else
                    testTarget[nTest] = ( REAL ) val * 0.001;
                //testTarget[nTest] = (REAL)val;
                if ( colCnt > 2 )
                    test[nTest*nFeat+colCnt-3] = ( REAL ) val * 0.001;
                //test[nTest*nFeat+colCnt-3] = (REAL)val;
            }
            pos++;
        }
        if ( ( colCnt-3 != nFeat-1 && addConstantOne == false ) || ( colCnt-3 != nFeat-2 && addConstantOne == true ) )
        {
            cout<<"colCnt:"<<colCnt<<" nFeat:"<<nFeat<<" addConstantOne:"<<addConstantOne<<endl;
            assert ( false );
        }
        if ( addConstantOne )
            test[nTest*nFeat+nFeat-1] = 1.0;
        nTest++;
    }
    fTestRMSE.close();

    /*
    // random subspace idea
    REAL subspace = 0.45;
    //REAL subspace = 1.0;
    bool* subspaceBit = new bool[nFeat];
    if(Framework::getFrameworkMode() == 0 && subspace < 1.0)  // training
    {
        //srand(time(0));
        cout<<"Create a random subspace:"<<subspace<<endl;
        fstream f((path+"/subspace.txt").c_str(), ios::out);
        for(int i=0;i<nFeat;i++)
        {
            subspaceBit[i] = (double)rand()/(double)RAND_MAX < subspace? true : false;
            subspaceBit[nFeat-1] = true;
            f<<(int)subspaceBit[i]<<endl;
            cout<<(int)subspaceBit[i]<<" ";
        }
        cout<<endl;
        f.close();
    }
    else if(subspace < 1.0) // prediction
    {
        cout<<"Read the random subspace"<<endl;
        fstream f((path+"/subspace.txt").c_str(), ios::in);
        for(int i=0;i<nFeat;i++)
        {
            f>>subspaceBit[i];
            cout<<(int)subspaceBit[i]<<" ";
        }
        cout<<endl;
        f.close();
    }

    if(subspace < 1.0)
    {
        int nFeatNew = 0;
        for(int i=0;i<nFeat;i++)
            nFeatNew += subspaceBit[i];
        cout<<"nFeatNew:"<<nFeatNew<<endl;

        REAL* trainNew = new REAL[nFeatNew*nTrain];
        REAL* testNew = new REAL[nFeatNew*nTest];

        for(int i=0;i<nTrain;i++)
        {
            int cnt = 0;
            for(int j=0;j<nFeat;j++)
            {
                if(subspaceBit[j])
                {
                    trainNew[cnt + i * nFeatNew] = train[j + i * nFeat];
                    cnt++;
                }
            }
        }

        for(int i=0;i<nTest;i++)
        {
            int cnt = 0;
            for(int j=0;j<nFeat;j++)
            {
                if(subspaceBit[j])
                {
                    testNew[cnt + i * nFeatNew] = test[j + i * nFeat];
                    cnt++;
                }
            }
        }

        delete[] train;
        delete[] test;
        train = trainNew;
        test = testNew;
        nFeat = nFeatNew;
    }
    */
    delete[] buf;
}

void DatasetReader::readAUSTRALIAN	(	string	path,
		REAL *&	train,
		REAL *&	trainTarget,
		int *&	trainLabel,
		REAL *&	test,
		REAL *&	testTarget,
		int *&	testLabel,
		uint &	nTrain,
		uint &	nTest,
		int &	nClass,
		int &	nDomain,
		int &	nFeat,
		REAL	positiveTarget = 1.0,
		REAL	negativeTarget = -1.0
	)

Reads the AUSTRALIAN dataset (UCI) 9 folders with 95 different signs in sum, 27 samples per sign

Definition at line 3307 of file DatasetReader.cpp.

{
    cout<<"Read AUSTRALIAN from: "<<path<<endl;
    nDomain = 1;

    char* dirs[] = {"tctodd1","tctodd2","tctodd3","tctodd4","tctodd5","tctodd6","tctodd7","tctodd8","tctodd9",0};

    char* signs[] = {"alive","all","answer","boy","building","buy","change_mind_","cold","come","computer_PC_","cost","crazy","danger","deaf","different","draw","drink","eat","exit","flash-light","forget","girl","give","glove","go","God","happy","head","hear","hello","his_hers","hot","how","hurry","hurt","I","innocent","is_true_","joke","juice","know","later","lose","love","make","man","maybe","mine","money","more","name","no","Norway","not-my-problem","paper","pen","please","polite","question","read","ready","research","responsible","right","sad","same","science","share","shop","soon","sorry","spend","stubborn","surprise","take","temper","thank","think","tray","us","voluntary","wait_notyet_","what","when","where","which","who","why","wild","will","write","wrong","yes","you","zero",0};

    nClass = 0;
    while ( signs[nClass] )
        nClass++;

    cout<<"nClass:"<<nClass<<endl;

    fstream fTrain;

    // get data bounds
    int nTrainTmp = 0;
    int dirCnt = 0;
    char buf[10000];
    int maxFrames = 0;
    int dataPerLine = 22;
    while ( dirs[dirCnt] )
    {
        int signCnt = 0;
        while ( signs[signCnt] )
        {
            for ( int i=0;i<3;i++ )
            {
                sprintf ( buf,"%s/%s/%s-%d.tsd",path.c_str(),dirs[dirCnt],signs[signCnt],i+1 );
                fTrain.open ( buf, ios::in );
                if ( fTrain.is_open() == false )
                    cout<<"Can not open "<<buf<<endl;
                else
                {
                    int lines = 0;
                    while ( fTrain.getline ( buf, 10000 ) ) // read all lines
                    {
                        stringstream ss ( buf );
                        REAL r;
                        int cnt = 0;
                        while ( ss>>r )
                            cnt++;
                        if ( cnt != dataPerLine )
                            assert ( false );
                        lines++;
                    }
                    if ( lines > maxFrames )
                        maxFrames = lines;
                    nTrainTmp++;
                }
                fTrain.close();
            }
            signCnt++;
        }
        dirCnt++;
    }

    cout<<"nTrainTmp:"<<nTrainTmp<<endl;
    cout<<"maxFrames:"<<maxFrames<<endl;

    nFeat = maxFrames * dataPerLine;
    cout<<"nFeat:"<<nFeat<<" ("<<maxFrames<<"*"<<dataPerLine<<")"<<endl;

    // allocate tmp mem
    REAL* trainTmp = new REAL[nTrainTmp * nFeat];
    int* trainLabelTmp = new int[nTrainTmp];
    for ( int i=0;i<nTrainTmp * nFeat;i++ )
        trainTmp[i] = 0.0;
    for ( int i=0;i<nTrainTmp;i++ )
        trainLabelTmp[i] = 0;

    // fill data
    nTrainTmp = 0;
    dirCnt = 0;
    while ( dirs[dirCnt] )
    {
        int signCnt = 0;
        while ( signs[signCnt] )
        {
            for ( int i=0;i<3;i++ )
            {
                sprintf ( buf,"%s/%s/%s-%d.tsd",path.c_str(),dirs[dirCnt],signs[signCnt],i+1 );
                fTrain.open ( buf, ios::in );
                if ( fTrain.is_open() == false )
                    cout<<"Can not open "<<buf<<endl;
                else
                {
                    int lines = 0;
                    while ( fTrain.getline ( buf, 10000 ) ) // read all lines
                    {
                        stringstream ss ( buf );
                        REAL r;
                        int cnt = 0;
                        while ( ss>>r )
                        {
                            trainTmp[nTrainTmp * nFeat + lines * dataPerLine + cnt] = r;
                            trainLabelTmp[nTrainTmp] = signCnt;
                            cnt++;
                        }
                        if ( cnt != dataPerLine )
                            assert ( false );
                        lines++;
                    }
                    nTrainTmp++;
                }
                fTrain.close();
            }
            signCnt++;
        }
        dirCnt++;
    }

    // split train and testset from trainTmp
    splitRandomTestset ( 0.2, trainTmp, trainLabelTmp, nTrainTmp, nFeat, nClass, train, trainLabel, trainTarget, test, testLabel, testTarget, nTrain, nTest, positiveTarget, negativeTarget );

    delete[] trainTmp;
    delete[] trainLabelTmp;

}

void DatasetReader::readAUSTRALIANCREDIT	(	string	path,
		REAL *&	train,
		REAL *&	trainTarget,
		int *&	trainLabel,
		REAL *&	test,
		REAL *&	testTarget,
		int *&	testLabel,
		uint &	nTrain,
		uint &	nTest,
		int &	nClass,
		int &	nDomain,
		int &	nFeat,
		REAL	positiveTarget = 1.0,
		REAL	negativeTarget = -1.0
	)

Reads the AUSTRALIAN-CREDIT dataset (UCI)

28735Bytes australian.dat 2467Bytes australian.doc

Definition at line 3553 of file DatasetReader.cpp.

{
    cout<<"Read AUSTRALIAN-CREDIT from: "<<path<<endl;
    nDomain = 1;

    // define data type and files
    int targetColumn = 15;
    uint nTrainTmp;
    char columnType[] = "dnndddnddnddnnd";
    char enabledCol[] = "111111111111111";
    const char* dataFiles[] = { ( new string ( path+"/australian.dat" ) )->c_str(),0};

    // === TRAIN SET ===
    getDataBounds ( dataFiles, " ", nFeat, nClass, nTrainTmp, columnType, enabledCol, targetColumn, 0 );

    // allocate tmp mem
    REAL* trainTmp = new REAL[nTrainTmp * nFeat];
    int* trainLabelTmp = new int[nTrainTmp];

    // fill data
    getDataBounds ( dataFiles, " ", nFeat, nClass, nTrainTmp, columnType, enabledCol, targetColumn, 0, true, trainTmp, trainLabelTmp );

    // split train and testset from trainTmp
    splitRandomTestset ( 0.2, trainTmp, trainLabelTmp, nTrainTmp, nFeat, nClass, train, trainLabel, trainTarget, test, testLabel, testTarget, nTrain, nTest, positiveTarget, negativeTarget );

    delete[] trainTmp;
    delete[] trainLabelTmp;
}

void DatasetReader::readBALANCE	(	string	path,
		REAL *&	train,
		REAL *&	trainTarget,
		int *&	trainLabel,
		REAL *&	test,
		REAL *&	testTarget,
		int *&	testLabel,
		uint &	nTrain,
		uint &	nTest,
		int &	nClass,
		int &	nDomain,
		int &	nFeat,
		REAL	positiveTarget = 1.0,
		REAL	negativeTarget = -1.0
	)

Reads the BALANCE dataset (UCI)

• 6250Bytes balance-scale.data
• 2222Bytes balance-scale.names

Definition at line 3436 of file DatasetReader.cpp.

{
    cout<<"Read BALANCE from: "<<path<<endl;
    nDomain = 1;

    // define data type and files
    int targetColumn = 1;
    uint nTrainTmp;
    char columnType[] = "dnnnn";
    char enabledCol[] = "11111";
    const char* dataFiles[] = { ( new string ( path+"/balance-scale.data" ) )->c_str(),0};

    // === TRAIN SET ===
    getDataBounds ( dataFiles, ",", nFeat, nClass, nTrainTmp, columnType, enabledCol, targetColumn, 0 );

    // allocate tmp mem
    REAL* trainTmp = new REAL[nTrainTmp * nFeat];
    int* trainLabelTmp = new int[nTrainTmp];

    // fill data
    getDataBounds ( dataFiles, ",", nFeat, nClass, nTrainTmp, columnType, enabledCol, targetColumn, 0, true, trainTmp, trainLabelTmp );

    // split train and testset from trainTmp
    splitRandomTestset ( 0.2, trainTmp, trainLabelTmp, nTrainTmp, nFeat, nClass, train, trainLabel, trainTarget, test, testLabel, testTarget, nTrain, nTest, positiveTarget, negativeTarget );

    delete[] trainTmp;
    delete[] trainLabelTmp;

}

void DatasetReader::readBINARY	(	string	path,
		REAL *&	train,
		REAL *&	trainTarget,
		int *&	trainLabel,
		REAL *&	test,
		REAL *&	testTarget,
		int *&	testLabel,
		uint &	nTrain,
		uint &	nTest,
		int &	nClass,
		int &	nDomain,
		int &	nFeat,
		REAL	positiveTarget = 1.0,
		REAL	negativeTarget = -1.0
	)

Reads a binary dataset format (result from feature selection) Limitied only to classification datasets

Format: [nExamples(4Byte INT),nClass(4Byte INT),nDomain(4Byte INT),nFeat(4Byte INT), features(nExamples*nFeat Bytes REAL),labels(nExamples*nDomain Bytes REAL)]

• binary.train
• binary.test

Definition at line 1818 of file DatasetReader.cpp.

{
    REAL* feat, *target;
    int* label, N;

    fstream f;
    if ( Framework::getFrameworkMode() == 1 )
        f.open ( ( path+"/binary.test" ).c_str(), ios::in );
    else
        f.open ( ( path+"/binary.train" ).c_str(), ios::in );

    // dataset bounds
    f.read ( ( char* ) &N, sizeof ( int ) );
    f.read ( ( char* ) &nClass, sizeof ( int ) );
    f.read ( ( char* ) &nDomain, sizeof ( int ) );
    f.read ( ( char* ) &nFeat, sizeof ( int ) );

    feat = new REAL[N*nFeat];
    target = new REAL[N*nClass*nDomain];
    label = new int[N*nDomain];

    // features and labels
    f.read ( ( char* ) feat, sizeof ( REAL ) *N*nFeat );
    f.read ( ( char* ) label, sizeof ( int ) *N*nDomain );
    f.close();

    for ( int i=0;i<N;i++ )
    {
        for ( int j=0;j<nClass*nDomain;j++ )
            target[i*nClass*nDomain+j] = negativeTarget;
        for ( int j=0;j<nDomain;j++ )
            target[i*nClass*nDomain + j*nClass + label[i*nDomain+j]] = positiveTarget;
    }

    if ( Framework::getFrameworkMode() == 1 )
    {
        nTest = N;
        test = feat;
        testTarget = target;
        testLabel = label;
        train = 0;
        trainTarget = 0;
        trainLabel = 0;
        nTrain = 0;
    }
    else
    {
        nTrain = N;
        train = feat;
        trainTarget = target;
        trainLabel = label;
        test = 0;
        testTarget = 0;
        testLabel = 0;
        nTest = 0;
    }

}

void DatasetReader::readBREASTCANCERWISCONSIN	(	string	path,
		REAL *&	train,
		REAL *&	trainTarget,
		int *&	trainLabel,
		REAL *&	test,
		REAL *&	testTarget,
		int *&	testLabel,
		uint &	nTrain,
		uint &	nTest,
		int &	nClass,
		int &	nDomain,
		int &	nFeat,
		REAL	positiveTarget = 1.0,
		REAL	negativeTarget = -1.0
	)

Reads the BEAST-CANCER-WISCONSIN dataset (UCI)

19889Bytes breast-cancer-wisconsin.data 5657Bytes breast-cancer-wisconsin.names 21363Bytes unformatted-data 124103Bytes wdbc.data 4708Bytes wdbc.names 44234Bytes wpbc.data 5671Bytes wpbc.names

Definition at line 3515 of file DatasetReader.cpp.

{
    cout<<"Read BREAST-CANCER-WISCONSIN from: "<<path<<endl;
    nDomain = 1;

    // define data type and files
    int targetColumn = 11;
    uint nTrainTmp;
    char columnType[] = "nnnnnnnnnnd";
    char enabledCol[] = "11111111111";
    //char columnType[] = "ddddddddddd";
    const char* dataFiles[] = { ( new string ( path+"/breast-cancer-wisconsin.data" ) )->c_str(),0};

    // === TRAIN SET ===
    getDataBounds ( dataFiles, ",", nFeat, nClass, nTrainTmp, columnType, enabledCol, targetColumn, 0 );

    // allocate tmp mem
    REAL* trainTmp = new REAL[nTrainTmp * nFeat];
    int* trainLabelTmp = new int[nTrainTmp];

    // fill data
    getDataBounds ( dataFiles, ",", nFeat, nClass, nTrainTmp, columnType, enabledCol, targetColumn, 0, true, trainTmp, trainLabelTmp );

    // split train and testset from trainTmp
    splitRandomTestset ( 0.2, trainTmp, trainLabelTmp, nTrainTmp, nFeat, nClass, train, trainLabel, trainTarget, test, testLabel, testTarget, nTrain, nTest, positiveTarget, negativeTarget );

    delete[] trainTmp;
    delete[] trainLabelTmp;

}

void DatasetReader::readCSV	(	string	path,
		REAL *&	train,
		REAL *&	trainTarget,
		int *&	trainLabel,
		REAL *&	test,
		REAL *&	testTarget,
		int *&	testLabel,
		uint &	nTrain,
		uint &	nTest,
		int &	nClass,
		int &	nDomain,
		int &	nFeat,
		REAL	positiveTarget = 1.0,
		REAL	negativeTarget = -1.0
	)

Regression+Classification Reads a csv dataset format, fields separated by delimiter

Definition at line 1882 of file DatasetReader.cpp.

{
    cout<<"Read CSV from: "<<path<<endl;
    nDomain = 1;
    
    char* buf = new char[1024*1024];
    char del = 0;
    int trainTargetColumn = -1;
    string trainName, testName;
    
    // read the settings file
    fstream fSetting(string(path+"/settings.txt").c_str(),ios::in);
    while ( fSetting.getline ( buf, 1024*1024 ) )
    {
        string s = buf;
        size_t pos = s.find_first_of('=');
        string token = s.substr(0,pos);
        //cout<<token<<endl;
        if(token == "delimiter")
            del = buf[pos+1];
        else if(token == "trainTargetColumn")
            trainTargetColumn = atoi(s.substr(pos+1).c_str());
        else if(token == "train")
            trainName = s.substr(pos+1);
        else if(token == "test")
            testName = s.substr(pos+1);
    }
    fSetting.close();
    
    // check if available
    if(trainTargetColumn == -1 || del == 0 || trainName == "" || (Framework::getFrameworkMode() && testName == ""))
        assert(false);
    
    // read training set
    fstream fTrain(string(path+"/"+trainName).c_str(),ios::in);
    vector<string> targets;
    map<string,int> targetMap;
    vector<vector<REAL> > features;
    while ( fTrain.getline ( buf, 1024*1024 ) )
    {
        string s = buf;
        size_t lastPos = 0;
        vector<REAL> feature;
        for(int i=0;i<s.length();i++)
        {
            if(s[i] == del || i == s.length()-1)
            {
                string token = s.substr(lastPos,i-lastPos);  // tokens from beginning
                if(i == s.length()-1)  // the last token in the line
                    token = s.substr(lastPos,i-lastPos+1);
                if(feature.size() == trainTargetColumn)  // any value
                {
                    targets.push_back(token);
                    if(Framework::getDatasetType())
                    {
                        map<string,int>::iterator it = targetMap.find(token);
                        if(it == targetMap.end())
                            targetMap[token] = targetMap.size();
                    }
                }
                else  // real value
                {
                    if((token[0] == '-' || token[0] == '.' || token[0] >= '0' && token[0] <= '9') == 0)  // real value check
                        assert(false);
                    REAL value = atof(token.c_str());
                    //cout<<value<<" ";
                    feature.push_back(value);
                }
                lastPos = i+1;
            }
        }
        if(feature.size())
            features.push_back(feature);
        //cout<<targets[targets.size()-1]<<endl;
    }
    fTrain.close();
    
    // count the different targets in a classification problem
    nClass = 1;
    if(Framework::getDatasetType())
    {
        nClass = targetMap.size();
        map<string,int>::iterator it;
        cout<<"Target values: ";
        for(it=targetMap.begin();it!=targetMap.end();it++)
            cout<<"["<<it->second<<"]"<<it->first<<" ";
        cout<<endl;
    }
    
    // assign bounds and allocate mem
    nTrain = features.size();
    nTest = 0;
    nFeat = features[0].size();
    train = new REAL[nFeat*nTrain];
    trainTarget = new REAL[nClass*nTrain];
    if(Framework::getDatasetType())
        trainLabel = new int[nTrain];
    
    // fill train data
    for(int i=0;i<nTrain;i++)
    {
        for(int j=0;j<nFeat;j++)  // fill features
            train[i*nFeat+j] = features[i][j];
        if(Framework::getDatasetType())  // classification dataset ?
        {
            int label = targetMap[targets[i]];
            trainLabel[i] = label;
            for(int j=0;j<nClass;j++)
                trainTarget[i*nClass+j] = (j==label? positiveTarget : negativeTarget);
        }
        else  // regression dataset
        {
            REAL target = atof(targets[i].c_str());
            trainTarget[i] = target;
        }
    }
    
    // read test set
    if(Framework::getFrameworkMode())
    {
        fstream fTest(string(path+"/"+testName).c_str(),ios::in);
        targets.clear();
        features.clear();
        while ( fTest.getline ( buf, 1024*1024 ) )
        {
            string s = buf;
            size_t lastPos = 0;
            vector<REAL> feature;
            for(int i=0;i<s.length();i++)
            {
                if(s[i] == del || i == s.length()-1)
                {
                    string token = s.substr(lastPos,i-lastPos);  // tokens from beginning
                    if(i == s.length()-1)  // the last token in the line
                        token = s.substr(lastPos,i-lastPos+1);
                    if(feature.size() == trainTargetColumn)  // any value
                    {
                        targets.push_back(token);
                        if(Framework::getDatasetType())
                        {
                            map<string,int>::iterator it = targetMap.find(token);
                            if(it == targetMap.end())
                                targetMap[token] = targetMap.size();
                        }
                    }
                    else  // real value
                    {
                        if((token[0] == '-' || token[0] == '.' || token[0] >= '0' && token[0] <= '9') == 0)  // real value check
                            assert(false);
                        REAL value = atof(token.c_str());
                        //cout<<value<<" ";
                        feature.push_back(value);
                    }
                    lastPos = i+1;
                }
            }
            if(feature.size())
                features.push_back(feature);
            //cout<<targets[targets.size()-1]<<endl;
        }
        fTest.close();
        
        // assign bounds and allocate mem
        nTest = features.size();
        test = new REAL[nFeat*nTest];
        testTarget = new REAL[nClass*nTest];
        if(Framework::getDatasetType())
            testLabel = new int[nTrain];
        
        // fill train data
        for(int i=0;i<nTest;i++)
        {
            for(int j=0;j<nFeat;j++)  // fill features
                test[i*nFeat+j] = features[i][j];
            if(targets.size() == features.size())
            {
                if(Framework::getDatasetType())  // classification dataset ?
                {
                    int label = targetMap[targets[i]];
                    testLabel[i] = label;
                    for(int j=0;j<nClass;j++)
                        testTarget[i*nClass+j] = (j==label? positiveTarget : negativeTarget);
                }
                else  // regression dataset
                {
                    REAL target = atof(targets[i].c_str());
                    testTarget[i] = target;
                }
            }
        }
        
    }
    
    delete[] buf;
}

void DatasetReader::readCYLINDERBANDS	(	string	path,
		REAL *&	train,
		REAL *&	trainTarget,
		int *&	trainLabel,
		REAL *&	test,
		REAL *&	testTarget,
		int *&	testLabel,
		uint &	nTrain,
		uint &	nTest,
		int &	nClass,
		int &	nDomain,
		int &	nFeat,
		REAL	positiveTarget = 1.0,
		REAL	negativeTarget = -1.0
	)

Reads the CYLINDER-BANDS dataset (UCI)

• 103300Bytes bands.data
• 3481Bytes bands.names

Definition at line 3473 of file DatasetReader.cpp.

{
    cout<<"Read CYLINDER-BANDS from: "<<path<<endl;
    nDomain = 1;

    // define data type and files
    int targetColumn = 40;
    uint nTrainTmp;
    char columnType[] = "ndddddddddddddddddddnnnnnnnnnnnnnnnnnnnd";
    char enabledCol[] = "1111111111111111111111111111111111111111";
    const char* dataFiles[] = { ( new string ( path+"/bands.data" ) )->c_str(),0};

    // === TRAIN SET ===
    getDataBounds ( dataFiles, ",", nFeat, nClass, nTrainTmp, columnType, enabledCol, targetColumn, 0 );

    // allocate tmp mem
    REAL* trainTmp = new REAL[nTrainTmp * nFeat];
    int* trainLabelTmp = new int[nTrainTmp];

    // fill data
    getDataBounds ( dataFiles, ",", nFeat, nClass, nTrainTmp, columnType, enabledCol, targetColumn, 0, true, trainTmp, trainLabelTmp );

    // split train and testset from trainTmp
    splitRandomTestset ( 0.2, trainTmp, trainLabelTmp, nTrainTmp, nFeat, nClass, train, trainLabel, trainTarget, test, testLabel, testTarget, nTrain, nTest, positiveTarget, negativeTarget );

    delete[] trainTmp;
    delete[] trainLabelTmp;

}

void DatasetReader::readDIABETES	(	string	path,
		REAL *&	train,
		REAL *&	trainTarget,
		int *&	trainLabel,
		REAL *&	test,
		REAL *&	testTarget,
		int *&	testLabel,
		uint &	nTrain,
		uint &	nTest,
		int &	nClass,
		int &	nDomain,
		int &	nFeat,
		REAL	positiveTarget = 1.0,
		REAL	negativeTarget = -1.0
	)

Reads the DIABETES dataset (UCI)

23279Bytes pima-indians-diabetes.data 3067Bytes pima-indians-diabetes.names

Definition at line 3589 of file DatasetReader.cpp.

{
    cout<<"Read DIABETES from: "<<path<<endl;
    nDomain = 1;

    // define data type and files
    int targetColumn = 9;
    uint nTrainTmp;
    char columnType[] = "nnnnnnnnd";
    char enabledCol[] = "111111111";
    const char* dataFiles[] = { ( new string ( path+"/pima-indians-diabetes.data" ) )->c_str(),0};

    // === TRAIN SET ===
    getDataBounds ( dataFiles, ",", nFeat, nClass, nTrainTmp, columnType, enabledCol, targetColumn, 0 );

    // allocate tmp mem
    REAL* trainTmp = new REAL[nTrainTmp * nFeat];
    int* trainLabelTmp = new int[nTrainTmp];

    // fill data
    getDataBounds ( dataFiles, ",", nFeat, nClass, nTrainTmp, columnType, enabledCol, targetColumn, 0, true, trainTmp, trainLabelTmp );

    // split train and testset from trainTmp
    splitRandomTestset ( 0.2, trainTmp, trainLabelTmp, nTrainTmp, nFeat, nClass, train, trainLabel, trainTarget, test, testLabel, testTarget, nTrain, nTest, positiveTarget, negativeTarget );

    delete[] trainTmp;
    delete[] trainLabelTmp;

}

void DatasetReader::readGERMAN	(	string	path,
		REAL *&	train,
		REAL *&	trainTarget,
		int *&	trainLabel,
		REAL *&	test,
		REAL *&	testTarget,
		int *&	testLabel,
		uint &	nTrain,
		uint &	nTest,
		int &	nClass,
		int &	nDomain,
		int &	nFeat,
		REAL	positiveTarget = 1.0,
		REAL	negativeTarget = -1.0
	)

Reads the GERMAN dataset (UCI)

79793Bytes german.data 102000Bytes german.data-numeric 4679Bytes german.doc

Definition at line 3626 of file DatasetReader.cpp.

{
    cout<<"Read GERMAN from: "<<path<<endl;
    nDomain = 1;

    // define data type and files
    int targetColumn = 21;
    uint nTrainTmp;
    char columnType[] = "dnddnddnddndnddndnddd";
    char enabledCol[] = "111111111111111111111";
    const char* dataFiles[] = { ( new string ( path+"/german.data" ) )->c_str(),0};

    // === TRAIN SET ===
    getDataBounds ( dataFiles, " ", nFeat, nClass, nTrainTmp, columnType, enabledCol, targetColumn, 0 );

    // allocate tmp mem
    REAL* trainTmp = new REAL[nTrainTmp * nFeat];
    int* trainLabelTmp = new int[nTrainTmp];

    // fill data
    getDataBounds ( dataFiles, " ", nFeat, nClass, nTrainTmp, columnType, enabledCol, targetColumn, 0, true, trainTmp, trainLabelTmp );

    // split train and testset from trainTmp
    splitRandomTestset ( 0.2, trainTmp, trainLabelTmp, nTrainTmp, nFeat, nClass, train, trainLabel, trainTarget, test, testLabel, testTarget, nTrain, nTest, positiveTarget, negativeTarget );

    delete[] trainTmp;
    delete[] trainLabelTmp;

}

void DatasetReader::readGLASS	(	string	path,
		REAL *&	train,
		REAL *&	trainTarget,
		int *&	trainLabel,
		REAL *&	test,
		REAL *&	testTarget,
		int *&	testLabel,
		uint &	nTrain,
		uint &	nTest,
		int &	nClass,
		int &	nDomain,
		int &	nFeat,
		REAL	positiveTarget = 1.0,
		REAL	negativeTarget = -1.0
	)

Reads the GLASS dataset (UCI)

11903Bytes glass.data 3506Bytes glass.names 780Bytes glass.tag

Definition at line 3663 of file DatasetReader.cpp.

{
    cout<<"Read GLASS from: "<<path<<endl;
    nDomain = 1;

    // define data type and files
    int targetColumn = 11;
    uint nTrainTmp;
    char columnType[] = "nnnnnnnnnnd";
    char enabledCol[] = "01111111111";
    const char* dataFiles[] = { ( new string ( path+"/glass.data" ) )->c_str(),0};

    // === TRAIN SET ===
    getDataBounds ( dataFiles, ",", nFeat, nClass, nTrainTmp, columnType, enabledCol, targetColumn, 0 );

    // allocate tmp mem
    REAL* trainTmp = new REAL[nTrainTmp * nFeat];
    int* trainLabelTmp = new int[nTrainTmp];

    // fill data
    getDataBounds ( dataFiles, ",", nFeat, nClass, nTrainTmp, columnType, enabledCol, targetColumn, 0, true, trainTmp, trainLabelTmp );

    // split train and testset from trainTmp
    splitRandomTestset ( 0.2, trainTmp, trainLabelTmp, nTrainTmp, nFeat, nClass, train, trainLabel, trainTarget, test, testLabel, testTarget, nTrain, nTest, positiveTarget, negativeTarget );

    delete[] trainTmp;
    delete[] trainLabelTmp;

}

void DatasetReader::readHEART	(	string	path,
		REAL *&	train,
		REAL *&	trainTarget,
		int *&	trainLabel,
		REAL *&	test,
		REAL *&	testTarget,
		int *&	testLabel,
		uint &	nTrain,
		uint &	nTest,
		int &	nClass,
		int &	nDomain,
		int &	nFeat,
		REAL	positiveTarget = 1.0,
		REAL	negativeTarget = -1.0
	)

Reads the HEART dataset (UCI)

4979Bytes SPECTF.names 33459Bytes SPECTF.test 10797Bytes SPECTF.train

Definition at line 3700 of file DatasetReader.cpp.

{
    cout<<"Read HEART from: "<<path<<endl;
    nDomain = 1;

    // define data type and files
    int targetColumn = 1;
    char columnType[] = "dnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn";
    char enabledCol[] = "111111111111111111111111111111111111111111111";
    const char* dataFiles[] = { ( new string ( path+"/SPECTF.train" ) )->c_str(), ( new string ( path+"/SPECTF.test" ) )->c_str(),0};

    // === TRAIN SET ===
    getDataBounds ( dataFiles, ",", nFeat, nClass, nTrain, columnType, enabledCol, targetColumn, 0 );
    train = new REAL[nFeat*nTrain];
    trainLabel = new int[nTrain];
    getDataBounds ( dataFiles, ",", nFeat, nClass, nTrain, columnType, enabledCol, targetColumn, 0, true, train, trainLabel );

    // === TEST SET ===
    getDataBounds ( dataFiles, ",", nFeat, nClass, nTest, columnType, enabledCol, targetColumn, 1 );
    test = new REAL[nFeat*nTest];
    testLabel = new int[nTest];
    getDataBounds ( dataFiles, ",", nFeat, nClass, nTest, columnType, enabledCol, targetColumn, 1, true, test, testLabel );

    // make numerical test targets
    makeNumericTrainAndTestTargets ( nClass, nTrain, nTest, positiveTarget, negativeTarget, trainLabel, testLabel, trainTarget, testTarget );

}

void DatasetReader::readHEPATITIS	(	string	path,
		REAL *&	train,
		REAL *&	trainTarget,
		int *&	trainLabel,
		REAL *&	test,
		REAL *&	testTarget,
		int *&	testLabel,
		uint &	nTrain,
		uint &	nTest,
		int &	nClass,
		int &	nDomain,
		int &	nFeat,
		REAL	positiveTarget = 1.0,
		REAL	negativeTarget = -1.0
	)

Reads the HEPATITIS dataset (UCI)

7545Bytes hepatitis.data 3098Bytes hepatitis.names

Definition at line 3734 of file DatasetReader.cpp.

{
    cout<<"Read HEPATITIS from: "<<path<<endl;
    nDomain = 1;

    // define data type and files
    int targetColumn = 1;
    uint nTrainTmp;
    char columnType[] = "dnnnnnnnnnnnnnnnnnnn";
    char enabledCol[] = "11111111111111111111";
    const char* dataFiles[] = { ( new string ( path+"/hepatitis.data" ) )->c_str(),0};

    // === TRAIN SET ===
    getDataBounds ( dataFiles, ",", nFeat, nClass, nTrainTmp, columnType, enabledCol, targetColumn, 0 );

    // allocate tmp mem
    REAL* trainTmp = new REAL[nTrainTmp * nFeat];
    int* trainLabelTmp = new int[nTrainTmp];

    // fill data
    getDataBounds ( dataFiles, ",", nFeat, nClass, nTrainTmp, columnType, enabledCol, targetColumn, 0, true, trainTmp, trainLabelTmp );

    // split train and testset from trainTmp
    splitRandomTestset ( 0.2, trainTmp, trainLabelTmp, nTrainTmp, nFeat, nClass, train, trainLabel, trainTarget, test, testLabel, testTarget, nTrain, nTest, positiveTarget, negativeTarget );

    delete[] trainTmp;
    delete[] trainLabelTmp;

}

void DatasetReader::readIONOSPHERE	(	string	path,
		REAL *&	train,
		REAL *&	trainTarget,
		int *&	trainLabel,
		REAL *&	test,
		REAL *&	testTarget,
		int *&	testLabel,
		uint &	nTrain,
		uint &	nTest,
		int &	nClass,
		int &	nDomain,
		int &	nFeat,
		REAL	positiveTarget = 1.0,
		REAL	negativeTarget = -1.0
	)

Reads the IONOSPHERE dataset (UCI)

76467Bytes ionosphere.data 3116Bytes ionosphere.names

Definition at line 3770 of file DatasetReader.cpp.

{
    cout<<"Read IONOSPHERE from: "<<path<<endl;
    nDomain = 1;

    // define data type and files
    int targetColumn = 35;
    uint nTrainTmp;
    char columnType[] = "nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnd";
    char enabledCol[] = "11111111111111111111111111111111111";
    const char* dataFiles[] = { ( new string ( path+"/ionosphere.data" ) )->c_str(),0};

    // === TRAIN SET ===
    getDataBounds ( dataFiles, ",", nFeat, nClass, nTrainTmp, columnType, enabledCol, targetColumn, 0 );

    // allocate tmp mem
    REAL* trainTmp = new REAL[nTrainTmp * nFeat];
    int* trainLabelTmp = new int[nTrainTmp];

    // fill data
    getDataBounds ( dataFiles, ",", nFeat, nClass, nTrainTmp, columnType, enabledCol, targetColumn, 0, true, trainTmp, trainLabelTmp );

    // split train and testset from trainTmp
    splitRandomTestset ( 0.2, trainTmp, trainLabelTmp, nTrainTmp, nFeat, nClass, train, trainLabel, trainTarget, test, testLabel, testTarget, nTrain, nTest, positiveTarget, negativeTarget );

    delete[] trainTmp;
    delete[] trainLabelTmp;

}

void DatasetReader::readIRIS	(	string	path,
		REAL *&	train,
		REAL *&	trainTarget,
		int *&	trainLabel,
		REAL *&	test,
		REAL *&	testTarget,
		int *&	testLabel,
		uint &	nTrain,
		uint &	nTest,
		int &	nClass,
		int &	nDomain,
		int &	nFeat,
		REAL	positiveTarget = 1.0,
		REAL	negativeTarget = -1.0
	)

Reads the IRIS dataset (UCI)

4551Bytes iris.data 2998Bytes iris.names

Definition at line 3807 of file DatasetReader.cpp.

{
    cout<<"Read IRIS from: "<<path<<endl;
    nDomain = 1;

    // define data type and files
    int targetColumn = 5;
    uint nTrainTmp;
    char columnType[] = "nnnnd";
    char enabledCol[] = "11111";
    const char* dataFiles[] = { ( new string ( path+"/iris.data" ) )->c_str(),0};

    // === TRAIN SET ===
    getDataBounds ( dataFiles, ",", nFeat, nClass, nTrainTmp, columnType, enabledCol, targetColumn, 0 );

    // allocate tmp mem
    REAL* trainTmp = new REAL[nTrainTmp * nFeat];
    int* trainLabelTmp = new int[nTrainTmp];

    // fill data
    getDataBounds ( dataFiles, ",", nFeat, nClass, nTrainTmp, columnType, enabledCol, targetColumn, 0, true, trainTmp, trainLabelTmp );

    // split train and testset from trainTmp
    splitRandomTestset ( 0.2, trainTmp, trainLabelTmp, nTrainTmp, nFeat, nClass, train, trainLabel, trainTarget, test, testLabel, testTarget, nTrain, nTest, positiveTarget, negativeTarget );

    delete[] trainTmp;
    delete[] trainLabelTmp;

}

void DatasetReader::readKDDCup09Large	(	string	path,
		REAL *&	train,
		REAL *&	trainTarget,
		int *&	trainLabel,
		REAL *&	test,
		REAL *&	testTarget,
		int *&	testLabel,
		uint &	nTrain,
		uint &	nTest,
		int &	nClass,
		int &	nDomain,
		int &	nFeat,
		REAL	positiveTarget = 1.0,
		REAL	negativeTarget = -1.0
	)

Reads the KNNCup09Large dataset

Definition at line 120 of file DatasetReader.cpp.

{
    if ( 0 )
    {
        readKDDCup09LargeBin ( path, train, trainTarget, trainLabel, test, testTarget, testLabel, nTrain, nTest, nClass, nDomain, nFeat, positiveTarget, negativeTarget );
        return;
    }

    time_t t0 = time ( 0 );

    nDomain = 3;

    cout<<"Read KDDCup09 from: "<<path<<endl;

    // 8-full-numeric inputs
    // (epoch=10) reg=0.00113318 .... [classErr:49.6833%] [rmse:0.99835] [probe:-0.498109]  [CalcBlend] lambda:0.001  [classErr:61.3693%] [rmse:1.12297] ERR=-0.648214 35[s] !min! [saveBest][SB]

    char* targetFiles[] =  //"orange_large_train_toy.labels",
    {
        "orange_large_train_churn.labels",
        "orange_large_train_appetency.labels",
        "orange_large_train_upselling.labels"
    };

    int nPreAlloc = 100000000;
    char *buf0 = new char[512], *buf1 = new char[512];
    char* lineBuf = new char[nPreAlloc];
    //REAL* support;
    //int* supportCnt;

    int NUM = 14740, CAT = 260, NLINES = 50000;
    int nFiles = 5;
    bool setNumZerosToMeans = false;
    bool setMissingToMeans = false;
    int numericMinMissing = 1;
    int numericMaxCluster = 0;  // add categoric (one-hot) from numeric input, max. occurence cnt
    int minAttributeOccurenceCategorical = 50*nFiles;  // 20
    int minAttributeOccurenceNumerical = 500*nFiles;  // 200
    REAL maxSTD = 1e10; // 10
    cout<<"nFiles:"<<nFiles<<" minAttrOccurCat:"<<minAttributeOccurenceCategorical<<" minAttrOccurNum:"<<minAttributeOccurenceNumerical<<endl;
    cout<<setNumZerosToMeans<<" "<<setMissingToMeans<<" "<<numericMaxCluster<<" "<<minAttributeOccurenceCategorical<<" "<<minAttributeOccurenceNumerical<<" "<<maxSTD<<endl;

    vector<string>* numericalAttributes = new vector<string>[NUM];
    vector<int>* numericalAttributesCnt = new vector<int>[NUM];
    vector<string>* categoricalAttributes = new vector<string>[CAT];
    vector<int>* categoricalAttributesCnt = new vector<int>[CAT];
    bool* categoricalHasMissingBin = new bool[CAT];
    int* categoricalMissingCnt = new int[CAT];
    bool* categoricalHasUnknownBin = new bool[CAT];
    for ( int i=0;i<CAT;i++ )
    {
        categoricalHasMissingBin[i] = false;
        categoricalHasUnknownBin[i] = false;
        categoricalMissingCnt[i] = 0;
    }
    int* numericNonZeroCnt = new int[NUM];
    int* numericMissingCnt = new int[NUM];
    bool* numericHasMissingBin = new bool[NUM];
    double* numericNonZeroPercent = new double[NUM];
    for ( int i=0;i<NUM;i++ )
    {
        numericMissingCnt[i] = 0;
        numericNonZeroCnt[i] = 0;
        numericNonZeroPercent[i] = 0.0;
        numericHasMissingBin[i] = false;
    }

    double* minValues = new double[100000];
    double* maxValues = new double[100000];
    double* maxNormValues = new double[100000];
    double* meanValues = new double[100000];
    double* stdValues = new double[100000];
    double* mean2Values = new double[100000];
    int* meanCnt = new int[100000];
    for ( int i=0;i<100000;i++ )
    {
        minValues[i] = 1e20;
        maxValues[i] = -1e20;
        maxNormValues[i] = 0.0;
        meanValues[i] = 0.0;
        mean2Values[i] = 0.0;
        meanCnt[i] = 0;
        stdValues[i] = 0.0;
    }

    //===========================================================================================================================
    //===========================================================================================================================
    // Loop over 2 states:
    // - State=0  read train values (+build index tables)
    // - State=1  store to features (train or test)
    //
    for ( int state=0;state<2;state++ )
    {
        int nTrainFill = 0;
        if ( state == 0 )
        {
            nTrain = 0;
        }

        //=======================================================================================================================
        //=======================================================================================================================
        // Loop over n files (file chunks)
        //
        for ( int file=0;file<nFiles;file++ )
        {
            // open train or test set
            if ( state == 0 )
                sprintf ( buf0,"%s/orange_large_train.data.chunk%d",path.c_str(), file+1 );
            else
            {
                if ( Framework::getFrameworkMode() == 1 )
                    sprintf ( buf0,"%s/orange_large_test.data.chunk%d",path.c_str(), file+1 );
                else
                    sprintf ( buf0,"%s/orange_large_train.data.chunk%d",path.c_str(), file+1 );
            }

            cout<<"Open:"<<buf0<<endl;
            fstream f;
            f.open ( buf0, ios::in );
            if ( f.is_open() == false )
                assert ( false );

            // read the first line in the first file (dummy)
            if ( file==0 )
                f.getline ( lineBuf, nPreAlloc );

            // tmp and count vars
            double zeroRatio = 0.0;
            double sparse = 0.0;
            int nTrainTmp = 0;

            //===================================================================================================================
            //===================================================================================================================
            // Read all lines of chunk file n
            //
            while ( f.getline ( lineBuf, nPreAlloc ) )
            {
                if ( nTrainTmp%1000 == 0 )
                    cout<<"."<<flush;

                // tmp and count vars
                int pos0 = 0, pos1 = 0;
                int nF = 0, nMissing = 0, nZeros = 0;
                int nFeatFill = 0;
                int nrHot = 0;
                double value;

                if ( state == 1 )
                {
                    // add constant one
                    train[nTrainFill*nFeat + nFeatFill] = 1.0;
                    nFeatFill++;
                }

                //===============================================================================================================
                //===============================================================================================================
                // Go through all characters of this line
                //
                while ( lineBuf[pos1] )
                {
                    // search for next tabulator
                    while ( lineBuf[pos1] != '\t' && lineBuf[pos1] != 0 )
                        pos1++;

                    //===========================================================================================================
                    //===========================================================================================================
                    // If the feature has some content
                    // This means no consecutive tabs
                    //
                    if ( pos1 > pos0 && lineBuf[pos1]!=0 )
                    {
                        // copy to tmp buffer
                        if ( pos1-pos0 <=0 || pos1-pos0 >= 512 )
                            assert ( false );
                        for ( int j=0;j<pos1-pos0;j++ )
                            buf1[j] = lineBuf[pos0+j];
                        buf1[pos1-pos0] = 0;


                        //=======================================================================================================
                        //=======================================================================================================
                        // Read Numeric value (feature count < NUM)
                        //
                        if ( nF < NUM )
                        {
                            if ( ( buf1[0]>='0' && buf1[0] <='9' ) || buf1[0]=='-' )
                                ;
                            else
                            {
                                cout<<"BUF:"<<buf1<<endl;
                                assert ( false );
                            }

                            //sscanf(buf1, "%f", &value);
                            value = atof ( buf1 );

                            if ( value == 0.0 )
                                nZeros++;

                            // first run through train data
                            if ( state==0 )
                            {
                                if ( minValues[nF] > value )
                                    minValues[nF] = value;
                                if ( maxValues[nF] < value )
                                    maxValues[nF] = value;

                                // histogram over numeric values
                                int size = numericalAttributes[nF].size();
                                if ( size < numericMaxCluster )
                                {
                                    int foundIndex = -1;
                                    for ( int j=0;j<size;j++ )
                                        if ( numericalAttributes[nF][j] == buf1 )
                                        {
                                            foundIndex = j;
                                            break;
                                        }
                                    // add value
                                    if ( foundIndex == -1 )
                                    {
                                        numericalAttributes[nF].push_back ( buf1 );
                                        numericalAttributesCnt[nF].push_back ( 1 );
                                    }
                                    else
                                        numericalAttributesCnt[nF][foundIndex]++;
                                }

                                if ( value != 0.0 )
                                {
                                    numericNonZeroCnt[nF]++;
                                    if ( numericNonZeroCnt[nF] > nTrain+nTrainTmp+1 )
                                    {
                                        cout<<"numericNonZeroCnt[nF]:"<<numericNonZeroCnt[nF]<<" nF:"<<nF<<" nTrainTmp:"<<nTrainTmp<<" nZeros:"<<nZeros<<" pos0:"<<pos0<<" pos1:"<<pos1<<endl;
                                        assert ( false );
                                    }
                                }

                                if ( value != 0.0 )
                                {
                                    // calc mean over numeric input
                                    meanValues[nF] += value;
                                    mean2Values[nF] += value * value;
                                    meanCnt[nF]++;
                                }
                            }
                            else if ( state==1 ) // second run, fill data tables
                            {
                                if ( numericNonZeroCnt[nF] >= minAttributeOccurenceNumerical && maxNormValues[nF] < stdValues[nF]*maxSTD )
                                {
                                    // numeric add
                                    if ( value == 0.0 && setNumZerosToMeans )
                                        train[nTrainFill*nFeat + nFeatFill] = meanValues[nF];
                                    else
                                        train[nTrainFill*nFeat + nFeatFill] = value;
                                    nFeatFill++;

                                    // numeric one hot add
                                    int size = numericalAttributes[nF].size();
                                    if ( size < numericMaxCluster && size > 1 )
                                    {
                                        int foundIndex = -1;
                                        for ( int j=0;j<size;j++ )
                                            if ( numericalAttributes[nF][j] == buf1 )
                                            {
                                                foundIndex = j;
                                                break;
                                            }
                                        // fill categorical
                                        int beforeHot = nrHot;
                                        for ( int j=0;j<size;j++ )
                                        {
                                            if ( foundIndex == j )
                                            {
                                                train[nTrainFill*nFeat + nFeatFill] = 1.0;
                                                nrHot++;
                                            }
                                            else
                                                train[nTrainFill*nFeat + nFeatFill] = 0.0;
                                            nFeatFill++;
                                        }
                                        // fill missing
                                        /*if(nrHot == beforeHot)
                                            train[nTrainFill*nFeat + nFeatFill] = 0.0;
                                        else
                                            train[nTrainFill*nFeat + nFeatFill] = 1.0;
                                        nFeatFill++;*/
                                    }

                                }

                                // missing values one-hot encoded
                                if ( numericHasMissingBin[nF] )
                                {
                                    train[nTrainFill*nFeat + nFeatFill] = 0.0;  // <- missing
                                    nFeatFill++;
                                    train[nTrainFill*nFeat + nFeatFill] = 1.0;  // <- available
                                    nFeatFill++;
                                }
                            }
                        }
                        //=======================================================================================================
                        //=======================================================================================================
                        // Read Categorical value (feature count >= NUM)
                        //
                        else
                        {
                            int index = nF-NUM;
                            if ( index >= CAT )
                                assert ( false );
                            int size = categoricalAttributes[index].size();
                            int sizeCnt = categoricalAttributesCnt[index].size();
                            if ( size != sizeCnt )
                                assert ( false );

                            int foundIndex = -1;
                            for ( int j=0;j<size;j++ )
                                if ( categoricalAttributes[index][j] == buf1 )
                                {
                                    foundIndex = j;
                                    break;
                                }

                            // first run through train data
                            if ( state==0 )
                            {
                                // add value
                                if ( foundIndex == -1 )
                                {
                                    categoricalAttributes[index].push_back ( buf1 );
                                    categoricalAttributesCnt[index].push_back ( 1 );
                                }
                                else // already exists
                                    categoricalAttributesCnt[index][foundIndex]++;
                            }
                            else if ( state==1 ) // second run, fill data tables
                            {
                                // one-hot encoding
                                int fillCnt = 0;
                                int beforeHot = nrHot;
                                for ( int j=0;j<size;j++ )
                                {
                                    if ( categoricalAttributesCnt[index][j] >= minAttributeOccurenceCategorical && categoricalAttributesCnt[index][j] < nTrain )
                                    {
                                        if ( foundIndex == j )
                                        {
                                            train[nTrainFill*nFeat + nFeatFill] = 1.0;
                                            nrHot++;
                                        }
                                        else
                                            train[nTrainFill*nFeat + nFeatFill] = 0.0;
                                        fillCnt++;
                                        nFeatFill++;
                                    }
                                }

                                // no missing (no consecutive tabs here)
                                if ( categoricalHasMissingBin[index] )
                                {
                                    train[nTrainFill*nFeat + nFeatFill] = 0.0;
                                    fillCnt++;
                                    nFeatFill++;
                                }

                                // if found, but not in cache
                                if ( categoricalHasUnknownBin[index] )
                                {
                                    if ( beforeHot == nrHot )
                                    {
                                        train[nTrainFill*nFeat + nFeatFill] = 1.0;
                                        nrHot++;
                                    }
                                    else
                                        train[nTrainFill*nFeat + nFeatFill] = 0.0;
                                    fillCnt++;
                                    nFeatFill++;
                                }

                                if ( nrHot != beforeHot + 1 && fillCnt > 0 )
                                {
                                    cout<<"WARNING: foundIndex:"<<foundIndex<<" "<<size<<" "<<index<<" "<<nrHot<<" "<<beforeHot<<" fill:"<<fillCnt<<endl;
                                    //assert(false);
                                }
                            }
                        }
                    }
                    //===========================================================================================================
                    //===========================================================================================================
                    // If the feature has no content
                    // Missing value here
                    //
                    else
                    {
                        nMissing++;

                        if ( state==0 )
                        {
                            // numeric
                            if ( nF < NUM )
                            {
                                numericMissingCnt[nF]++;
                            }
                            // categorical
                            if ( nF >= NUM )
                            {
                                int index = nF-NUM;
                                categoricalMissingCnt[index]++;
                            }
                        }

                        // second run, fill data tables with zeros
                        if ( state==1 )
                        {
                            //===================================================================================================
                            //===================================================================================================
                            // Read Numeric value (feature count < NUM)
                            //
                            if ( nF < NUM )
                            {
                                if ( numericNonZeroCnt[nF] >= minAttributeOccurenceNumerical && maxNormValues[nF] < stdValues[nF]*maxSTD )
                                {
                                    // numeric add
                                    if ( setMissingToMeans )
                                        train[nTrainFill*nFeat + nFeatFill] = meanValues[nF];
                                    else
                                        train[nTrainFill*nFeat + nFeatFill] = 0.0;
                                    nFeatFill++;

                                    // numeric one hot add
                                    int size = numericalAttributes[nF].size();
                                    if ( size < numericMaxCluster && size > 1 )
                                    {
                                        // fill categorical
                                        for ( int j=0;j<size;j++ )
                                        {
                                            train[nTrainFill*nFeat + nFeatFill] = 0.0;
                                            nFeatFill++;
                                        }
                                        // fill missing
                                        //train[nTrainFill*nFeat + nFeatFill] = 1.0;
                                        //nFeatFill++;
                                    }
                                }

                                // missing values one-hot encoded
                                if ( numericHasMissingBin[nF] )
                                {
                                    train[nTrainFill*nFeat + nFeatFill] = 1.0;  // <- missing
                                    nFeatFill++;
                                    train[nTrainFill*nFeat + nFeatFill] = 0.0;  // <- available
                                    nFeatFill++;
                                }
                            }
                            //===================================================================================================
                            //===================================================================================================
                            // Read Categorical value (feature count >= NUM)
                            //
                            else
                            {
                                int index = nF - NUM;
                                if ( index >= CAT )
                                    assert ( false );
                                int size = categoricalAttributes[index].size();
                                int sizeCnt = categoricalAttributesCnt[index].size();
                                if ( size != sizeCnt )
                                    assert ( false );

                                // one-hot encoding
                                int fillCnt = 0;
                                int beforeHot = nrHot;
                                for ( int j=0;j<size;j++ )
                                {
                                    if ( categoricalAttributesCnt[index][j] >= minAttributeOccurenceCategorical && categoricalAttributesCnt[index][j] < nTrain )
                                    {
                                        train[nTrainFill*nFeat + nFeatFill] = 0.0;  // no here
                                        fillCnt++;
                                        nFeatFill++;
                                    }
                                }
                                if ( categoricalHasMissingBin[index] )
                                {
                                    if ( fillCnt == 0 && categoricalHasUnknownBin[index] == false )
                                    {
                                        cout<<"categoricalMissingCnt["<<index<<"]:"<<categoricalMissingCnt[index]<<endl;
                                        assert ( false );
                                    }
                                    // set the input to "missing value"
                                    train[nTrainFill*nFeat + nFeatFill] = 1.0;
                                    nrHot++;
                                    fillCnt++;
                                    nFeatFill++;
                                }

                                if ( categoricalHasUnknownBin[index] )
                                {
                                    // no unknown value
                                    train[nTrainFill*nFeat + nFeatFill] = 0.0;
                                    fillCnt++;
                                    nFeatFill++;
                                }

                                if ( nrHot != beforeHot + 1 && fillCnt > 0 )
                                {
                                    cout<<"WARNING: "<<size<<" "<<index<<" "<<nrHot<<" "<<beforeHot<<" fill:"<<fillCnt<<endl;
                                    //assert(false);
                                }
                            }
                        }
                    }

                    // check for last character
                    if ( lineBuf[pos1]!=0 )
                        pos1++;

                    // beginpos = endpos
                    pos0 = pos1;

                    // column count
                    nF++;
                }

                // valid checks
                if ( nF != NUM + CAT )
                    assert ( false );
                if ( state==1 )
                {
                    if ( nFeatFill != nFeat )
                    {
                        cout<<"nFeatFill:"<<nFeatFill<<" nFeat:"<<nFeat<<endl;
                        assert ( false );
                    }
                    nTrainFill++;
                }

                nTrainTmp++;

                sparse += nMissing / ( double ) nF;
                zeroRatio += nZeros / ( double ) nF;
            }

            f.close();

            // ratio of sparseness and zeroPercent
            sparse /= ( double ) nTrainTmp;
            zeroRatio /= ( double ) nTrainTmp;
            cout<<"nTrainTmp:"<<nTrainTmp<<endl;
            cout<<"missing values:"<<100.0*sparse<<"%"<<endl;
            cout<<"zero values:"<<100.0*zeroRatio<<"%"<<endl;

            double min0 = 1e20, max0 = -1e20;
            for ( int i=0;i<100000;i++ )
            {
                if ( min0 > minValues[i] )
                    min0 = minValues[i];
                if ( max0 < maxValues[i] )
                    max0 = maxValues[i];
            }
            cout<<"min|max values: "<<min0<<"|"<<max0<<endl;

            int sum = 0;
            for ( int j=0;j<CAT;j++ )
                sum += categoricalAttributes[j].size();
            cout<<"nCategoricalSum:"<<sum<<endl;

            if ( state == 0 )
                nTrain += nTrainTmp;

        }

        // do some checks
        if ( state == 1 )
        {
            if ( nTrain != nTrainFill )
                assert ( false );

            for ( int i=0;i<nTrain*nFeat;i++ )
                if ( train[i] == 1e10 )
                {
                    cout<<"i:"<<i<<endl;
                    assert ( false );
                }
        }

        if ( state==0 )
        {
            for ( int i=0;i<NUM;i++ )
                numericNonZeroPercent[i] = ( double ) numericNonZeroCnt[i]/ ( double ) nTrain;
            for ( int i=0;i<100000;i++ )
                if ( meanCnt[i] > 0 )
                {
                    meanValues[i] /= ( double ) meanCnt[i];
                    stdValues[i] = sqrt ( mean2Values[i]/ ( double ) meanCnt[i] - meanValues[i]/ ( double ) meanCnt[i] );
                    maxNormValues[i] = fabs ( maxValues[i] - meanValues[i] );
                    if ( maxNormValues[i] < fabs ( minValues[i] - meanValues[i] ) )
                        maxNormValues[i] = fabs ( minValues[i] - meanValues[i] );
                }

            cout<<"nTrain:"<<nTrain<<endl;

            // === Calculate effective number of input features ===
            nFeat = 1; // const
            int nFeatNum = 0, nFeatNumRaw = 0, nFeatNumCat = 0, nFeatCat = 0, nUnknown = 0, nMissing = 0, nIn = 0, nNumMiss = 0;
            // numerical
            for ( int j=0;j<NUM;j++ )
            {
                if ( numericNonZeroCnt[j] >= minAttributeOccurenceNumerical && maxNormValues[j] < stdValues[j]*maxSTD )
                {
                    // standard numerical input
                    nFeat++;
                    nFeatNum++;
                    nFeatNumRaw++;

                    // numerical input with limited number of different values -> translate it to categorical input
                    if ( numericalAttributes[j].size() < numericMaxCluster && numericalAttributes[j].size() > 1 )
                    {
                        cout<<"nFeatNum:"<<nFeatNum<<" ";
                        for ( int k=0;k<numericalAttributes[j].size();k++ )
                        {
                            cout<<numericalAttributes[j][k]<<"("<<numericalAttributesCnt[j][k]<<") ";
                            nFeat++;
                            nFeatNum++;
                            nFeatNumCat++;
                        }
                        cout<<endl;
                        /*
                        // add one bin for "missing or unknown value"
                        nFeat++;
                        nFeatNum++;
                        nFeatNumCat++;*/
                    }
                    if ( numericMissingCnt[j] >= numericMinMissing )
                    {
                        numericHasMissingBin[j] = true;
                        nFeat+=2;
                        nNumMiss+=2;
                    }
                }
            }
            // categorical
            for ( int j=0;j<CAT;j++ )
            {
                int nUsed = 0, nUn = 0, nCat = 0, nMiss = 0, nUnk = 0;
                for ( int k=0;k<categoricalAttributesCnt[j].size();k++ )
                {
                    // count valid entries (with enough occurence)
                    if ( categoricalAttributesCnt[j][k] >= minAttributeOccurenceCategorical && categoricalAttributesCnt[j][k] < nTrain )
                    {
                        nFeat++;
                        nFeatCat++;
                        nUsed++;
                        nIn++;
                        nCat++;
                    }
                    else if ( categoricalAttributesCnt[j][k] < nTrain ) // not enough occurence -> put to unknown
                        nUn++;
                }
                // missing is like a normal categoric input
                if ( ( categoricalMissingCnt[j] >= minAttributeOccurenceCategorical && categoricalMissingCnt[j] < nTrain ) || categoricalMissingCnt[j] > 0 && nCat > 0 )
                {
                    // add a "missing value" input of this feature
                    nFeat++;
                    nFeatCat++;
                    nMissing++;
                    nMiss++;
                    categoricalHasMissingBin[j] = true;
                }
                if ( nUn > 0 && nCat + nMiss > 0 )
                {
                    // add a "unknown value" input of this feature
                    nFeat++;
                    nFeatCat++;
                    nUnknown++;
                    nUnk++;
                    categoricalHasUnknownBin[j] = true;
                }

                if ( nCat + nMiss + nUnk == 1 )
                    assert ( false );
            }

            cout<<"nFeat:"<<nFeat<<" (numInputs:"<<nFeatNum<<" [rawNum:"<<nFeatNumRaw<<" nFeatNumCat:"<<nFeatNumCat<<"] catInputs:"<<nFeatCat<<" [nUnknown:"<<nUnknown<<" nMissing:"<<nMissing<<" nCat:"<<nIn<<"] numMissingHot:"<<nNumMiss<<" [+1const.])"<<endl;

            cout<<"Allocate train features: "<< ( double ) nTrain*nFeat/1e6*4.0<<" MB"<<endl;
            train = new REAL[nTrain*nFeat];
            for ( int i=0;i<nTrain*nFeat;i++ )
                train[i] = 1e10;

            //support = new REAL[nFeat];
            //supportCnt = new int[nFeat];
            //for(int i=0;i<nFeat;i++)
            //{
            //support[i] = 0.0;
            //supportCnt[i] = 0;
            //}

            // read targets
            nClass = 2;
            trainTarget = new REAL[nTrain*nClass*nDomain];
            trainLabel = new int[nTrain*nDomain];
            for ( int d=0;d<nDomain;d++ )
            {
                sprintf ( buf0,"%s/%s",path.c_str(),targetFiles[d] );
                fstream f;
                cout<<"Open targets:"<<buf0<<endl;
                f.open ( buf0,ios::in );
                if ( f.is_open() == false )
                    assert ( false );
                int label;
                for ( int i=0;i<nTrain;i++ )
                {
                    f>>label;
                    if ( label==-1 )
                    {
                        trainTarget[i*nClass*nDomain + d*nClass + 0] = positiveTarget;
                        trainTarget[i*nClass*nDomain + d*nClass + 1] = negativeTarget;
                        trainLabel[i*nDomain + d] = 0;
                    }
                    else if ( label==1 )
                    {
                        trainTarget[i*nClass*nDomain + d*nClass + 0] = negativeTarget;
                        trainTarget[i*nClass*nDomain + d*nClass + 1] = positiveTarget;
                        trainLabel[i*nDomain + d] = 1;
                    }
                    else
                        assert ( false );
                }
                f.close();
            }
            // test set
            nTest = 0;
            test = 0;
            testTarget = 0;
            testLabel = 0;

        }
    }

    for ( int i=0;i<nTrain;i++ )
        for ( int j=0;j<nFeat;j++ )
            if ( train[i*nFeat+j] == 1e10 )
            {
                cout<<"i:"<<i<<" j:"<<j<<" "<<train[i*nFeat+j]<<endl;
                assert ( false );
            }


    fstream f;
    /*f.open("AAA.txt",ios::out);
    f<<"========= numerical ========="<<endl<<endl;
    for(int i=0;i<NUM;i++)
        if(numericNonZeroCnt[i] >= minAttributeOccurenceNumerical  && maxNormValues[i] < stdValues[i]*5.0)
            f<<i<<":"<<numericNonZeroCnt[i]<<"["<<numericNonZeroCnt[i]<<"]["<<minValues[i]<<"|"<<maxValues[i]<<"]"<<endl;
    f<<endl<<endl;
    f<<"========= categorical ========="<<endl<<endl;
    for(int i=0;i<CAT;i++)
    {
        int size = categoricalAttributes[i].size();

        int chkCnt = 0;
        for(int j=0;j<size;j++)
        if(categoricalAttributesCnt[i][j] >= minAttributeOccurenceCategorical && categoricalAttributesCnt[i][j] < NLINES)
                chkCnt++;

        if(chkCnt > 0)
            f<<endl<<"Attrib."<<i<<"(#"<<chkCnt<<"):";

        // go over all possible values
        for(int j=0;j<size;j++)
        {
            // find the max support
            int ind = -1;
            int max = -1;
            for(int k=0;k<size;k++)
            {
                if(categoricalAttributesCnt[i][k] > max)
                {
                    max = categoricalAttributesCnt[i][k];
                    ind = k;
                }
            }
            if(ind==-1)
                assert(false);

            if(categoricalAttributesCnt[i][ind] >= minAttributeOccurenceCategorical && categoricalAttributesCnt[i][ind] < NLINES)
                f<<j<<":"<<categoricalAttributes[i][ind]<<"["<<categoricalAttributesCnt[i][ind]<<"]("<<ind<<")  ";

            // mark as viewed
            categoricalAttributesCnt[i][ind] = -1;
        }

        if(categoricalHasMissingBin[i])
            f<<"+1xMissing"<<"[]("<<-1<<")  ";

        if(chkCnt > 0)
            f<<endl;

    }
    f.close();
    */
    if ( lineBuf )
    {
        delete[] lineBuf;
        lineBuf = 0;
    }
    if ( numericNonZeroCnt )
    {
        delete[] numericNonZeroCnt;
        numericNonZeroCnt = 0;
    }
    if ( numericNonZeroPercent )
    {
        delete[] numericNonZeroPercent;
        numericNonZeroPercent = 0;
    }
    if ( categoricalAttributes )
    {
        delete[] categoricalAttributes;
        categoricalAttributes = 0;
    }
    if ( meanValues )
    {
        delete[] meanValues;
        meanValues = 0;
    }
    if ( meanCnt )
    {
        delete[] meanCnt;
        meanCnt = 0;
    }
    if ( categoricalHasMissingBin )
    {
        delete[] categoricalHasMissingBin;
        categoricalHasMissingBin = 0;
    }

    // tmp print out of: train data
    f.open ( "A.txt",ios::out );
    double* mu = new double[nFeat];
    for ( int i=0;i<nFeat;i++ )
        mu[i] = 0.0;
    for ( int i=0;i<nTrain;i++ )
        for ( int j=0;j<nFeat;j++ )
            mu[j] += train[i*nFeat + j];
    for ( int i=0;i<nFeat;i++ )
        mu[i] /= ( double ) nTrain;
    for ( int i=0;i<nFeat;i++ )
        f<<mu[i]<<endl;
    f.close();

    //f.open("A.dat",ios::out);
    //f.write((char*)train,sizeof(REAL)*nTrain*nFeat);
    //f.close();

    /*f.open("A.txt",ios::out);
    for(int i=0;i<nTrain;i++)
    {
        for(int j=0;j<nFeat;j++)
            f<<train[i*nFeat+j]<<" ";
        f<<endl;
    }
    f.close();

    f.open("B.txt",ios::out);
    for(int i=0;i<nTrain;i++)
    {
        for(int j=0;j<nDomain;j++)
            f<<trainLabel[i*nDomain+j]<<" ";
        f<<endl;
    }
    f.close();

    f.open("C.txt",ios::out);
    for(int i=0;i<nTrain;i++)
    {
        for(int j=0;j<nDomain*nClass;j++)
            f<<trainTarget[i*nDomain*nClass+j]<<" ";
        f<<endl;
    }
    f.close();
    */
    if ( Framework::getFrameworkMode() == 1 )
    {
        cout<<endl<<"Set test data (and set train data to 0)"<<endl<<endl;
        test = train;
        train = 0;
        nTest = nTrain;
        nTrain = 0;
        testTarget = trainTarget;
        trainTarget = 0;
        testLabel = trainLabel;
        trainLabel = 0;
    }
    cout<<endl<<"Finished read in "<<time ( 0 )-t0<<"[s]"<<endl<<endl;
}

void DatasetReader::readKDDCup09LargeBin	(	string	path,
		REAL *&	train,
		REAL *&	trainTarget,
		int *&	trainLabel,
		REAL *&	test,
		REAL *&	testTarget,
		int *&	testLabel,
		uint &	nTrain,
		uint &	nTest,
		int &	nClass,
		int &	nDomain,
		int &	nFeat,
		REAL	positiveTarget = 1.0,
		REAL	negativeTarget = -1.0
	)

Read the large dataset from the KDDCup2009 (internal binary type)

Definition at line 24 of file DatasetReader.cpp.

{
    nDomain = 3;
    char* targetFiles[] =  //"orange_large_train_toy.labels",
    {
        "orange_large_train_churn.labels",
        "orange_large_train_appetency.labels",
        "orange_large_train_upselling.labels"
    };

    nTrain = 50000;
    nFeat = 113;

    char buf[512];
    if ( Framework::getFrameworkMode() == 1 )
    {
        //sprintf(buf,"featureSelection_churn_test_%d_features.dat",nFeat);
        //sprintf(buf,"featureSelection_appetency_test_%d_features.dat",nFeat);
        //sprintf(buf,"featureSelection_upselling_test_%d_features.dat",nFeat);
        sprintf ( buf,"featureSelection_all_test_%d_features.dat",nFeat );
    }
    else
    {
        //sprintf(buf,"featureSelection_churn_train_%d_features.dat",nFeat);
        //sprintf(buf,"featureSelection_appetency_train_%d_features.dat",nFeat);
        //sprintf(buf,"featureSelection_upselling_train_%d_features.dat",nFeat);
        sprintf ( buf,"featureSelection_all_train_%d_features.dat",nFeat );
    }
    cout<<"Open:"<<buf<<endl;
    train = new REAL[nTrain * nFeat];
    fstream f ( buf,ios::in );
    if ( f.is_open() == false )
        assert ( false );
    f.read ( ( char* ) train, sizeof ( REAL ) *nTrain*nFeat );
    f.close();

    // read targets
    nClass = 2;
    trainTarget = new REAL[nTrain*nClass*nDomain];
    trainLabel = new int[nTrain*nDomain];
    char buf0[512];
    for ( int d=0;d<nDomain;d++ )
    {
        sprintf ( buf0,"%s/%s",path.c_str(),targetFiles[d] );
        fstream f;
        cout<<"Open targets:"<<buf0<<endl;
        f.open ( buf0,ios::in );
        if ( f.is_open() == false )
            assert ( false );
        int label;
        for ( int i=0;i<nTrain;i++ )
        {
            f>>label;
            if ( label==-1 )
            {
                trainTarget[i*nClass*nDomain + d*nClass + 0] = positiveTarget;
                trainTarget[i*nClass*nDomain + d*nClass + 1] = negativeTarget;
                trainLabel[i*nDomain + d] = 0;
            }
            else if ( label==1 )
            {
                trainTarget[i*nClass*nDomain + d*nClass + 0] = negativeTarget;
                trainTarget[i*nClass*nDomain + d*nClass + 1] = positiveTarget;
                trainLabel[i*nDomain + d] = 1;
            }
            else
                assert ( false );
        }
        f.close();
    }

    // test set
    nTest = 0;
    test = 0;
    testTarget = 0;
    testLabel = 0;

    if ( Framework::getFrameworkMode() == 1 )
    {
        cout<<endl<<"Set test data (and set train data to 0)"<<endl<<endl;
        test = train;
        train = 0;
        nTest = nTrain;
        nTrain = 0;
        testTarget = trainTarget;
        trainTarget = 0;
        testLabel = trainLabel;
        trainLabel = 0;
    }

}

void DatasetReader::readKDDCup09Small	(	string	path,
		REAL *&	train,
		REAL *&	trainTarget,
		int *&	trainLabel,
		REAL *&	test,
		REAL *&	testTarget,
		int *&	testLabel,
		uint &	nTrain,
		uint &	nTest,
		int &	nClass,
		int &	nDomain,
		int &	nFeat,
		REAL	positiveTarget = 1.0,
		REAL	negativeTarget = -1.0
	)

Reads the KNNCup09Small dataset

Definition at line 1020 of file DatasetReader.cpp.

{
    time_t t0 = time ( 0 );

    nDomain = 3;

    cout<<"Read KDDCup09 from: "<<path<<endl;

    char* targetFiles[] = {"orange_small_train_churn.labels"
                           ,"orange_small_train_appetency.labels"
                           ,"orange_small_train_upselling.labels"
                          };

    int nPreAlloc = 100000000;
    char *buf0 = new char[512], *buf1 = new char[512];
    char* lineBuf = new char[nPreAlloc];

    int NUM = 190, CAT = 40, NLINES = 50000;
    int nFiles = 1;
    bool setNumZerosToMeans = false;
    bool setMissingToMeans = false;
    int numericMinMissing = 1;
    int numericMaxCluster = 0;  // add categoric (one-hot) from numeric input, max. occurence cnt
    int minAttributeOccurenceCategorical = 200*nFiles;  // 20
    int minAttributeOccurenceNumerical = 500*nFiles;  // 50
    REAL maxSTD = 1e10; // 10
    cout<<"nFiles:"<<nFiles<<" minAttrOccurCat:"<<minAttributeOccurenceCategorical<<" minAttrOccurNum:"<<minAttributeOccurenceNumerical<<endl;
    cout<<setNumZerosToMeans<<" "<<setMissingToMeans<<" "<<numericMaxCluster<<" "<<minAttributeOccurenceCategorical<<" "<<minAttributeOccurenceNumerical<<" "<<maxSTD<<endl;

    vector<string>* numericalAttributes = new vector<string>[NUM];
    vector<int>* numericalAttributesCnt = new vector<int>[NUM];
    vector<string>* categoricalAttributes = new vector<string>[CAT];
    vector<int>* categoricalAttributesCnt = new vector<int>[CAT];
    bool* categoricalHasMissingBin = new bool[CAT];
    int* categoricalMissingCnt = new int[CAT];
    bool* categoricalHasUnknownBin = new bool[CAT];
    for ( int i=0;i<CAT;i++ )
    {
        categoricalHasMissingBin[i] = false;
        categoricalHasUnknownBin[i] = false;
        categoricalMissingCnt[i] = 0;
    }
    int* numericNonZeroCnt = new int[NUM];
    int* numericMissingCnt = new int[NUM];
    bool* numericHasMissingBin = new bool[NUM];
    double* numericNonZeroPercent = new double[NUM];
    for ( int i=0;i<NUM;i++ )
    {
        numericMissingCnt[i] = 0;
        numericNonZeroCnt[i] = 0;
        numericNonZeroPercent[i] = 0.0;
        numericHasMissingBin[i] = false;
    }

    double* minValues = new double[100000];
    double* maxValues = new double[100000];
    double* maxNormValues = new double[100000];
    double* meanValues = new double[100000];
    double* stdValues = new double[100000];
    double* mean2Values = new double[100000];
    int* meanCnt = new int[100000];
    for ( int i=0;i<100000;i++ )
    {
        minValues[i] = 1e20;
        maxValues[i] = -1e20;
        maxNormValues[i] = 0.0;
        meanValues[i] = 0.0;
        mean2Values[i] = 0.0;
        meanCnt[i] = 0;
        stdValues[i] = 0.0;
    }

    //===========================================================================================================================
    //===========================================================================================================================
    // Loop over 2 states:
    // - State=0  read train values (+build index tables)
    // - State=1  store to features (train or test)
    //
    for ( int state=0;state<2;state++ )
    {
        int nTrainFill = 0;
        if ( state == 0 )
        {
            nTrain = 0;
        }

        //=======================================================================================================================
        //=======================================================================================================================
        // Loop over n files (file chunks)
        //
        for ( int file=0;file<nFiles;file++ )
        {
            // open train or test set
            if ( state == 0 )
                sprintf ( buf0,"%s/orange_small_train.data",path.c_str() );
            else
            {
                if ( Framework::getFrameworkMode() == 1 )
                    sprintf ( buf0,"%s/orange_small_test.data",path.c_str() );
                else
                    sprintf ( buf0,"%s/orange_small_train.data",path.c_str() );
            }

            cout<<"Open:"<<buf0<<endl;
            fstream f;
            f.open ( buf0, ios::in );
            if ( f.is_open() == false )
                assert ( false );

            // read the first line in the first file (dummy)
            if ( file==0 )
                f.getline ( lineBuf, nPreAlloc );

            // tmp and count vars
            double zeroRatio = 0.0;
            double sparse = 0.0;
            int nTrainTmp = 0;

            //===================================================================================================================
            //===================================================================================================================
            // Read all lines of chunk file n
            //
            while ( f.getline ( lineBuf, nPreAlloc ) )
            {
                if ( nTrainTmp%1000 == 0 )
                    cout<<"."<<flush;

                // tmp and count vars
                int pos0 = 0, pos1 = 0;
                int nF = 0, nMissing = 0, nZeros = 0;
                int nFeatFill = 0;
                int nrHot = 0;
                double value;

                if ( state == 1 )
                {
                    // add constant one
                    train[nTrainFill*nFeat + nFeatFill] = 1.0;
                    nFeatFill++;
                }

                //===============================================================================================================
                //===============================================================================================================
                // Go through all characters of this line
                //
                while ( lineBuf[pos1] )
                {
                    // search for next tabulator
                    while ( lineBuf[pos1] != '\t' && lineBuf[pos1] != 0 )
                        pos1++;

                    //===========================================================================================================
                    //===========================================================================================================
                    // If the feature has some content
                    // This means no consecutive tabs
                    //
                    if ( pos1 > pos0 && lineBuf[pos1]!=0 )
                    {
                        // copy to tmp buffer
                        if ( pos1-pos0 <=0 || pos1-pos0 >= 512 )
                            assert ( false );
                        for ( int j=0;j<pos1-pos0;j++ )
                            buf1[j] = lineBuf[pos0+j];
                        buf1[pos1-pos0] = 0;


                        //=======================================================================================================
                        //=======================================================================================================
                        // Read Numeric value (feature count < NUM)
                        //
                        if ( nF < NUM )
                        {
                            if ( ( buf1[0]>='0' && buf1[0] <='9' ) || buf1[0]=='-' )
                                ;
                            else
                            {
                                cout<<"BUF:"<<buf1<<endl;
                                assert ( false );
                            }

                            //sscanf(buf1, "%f", &value);
                            value = atof ( buf1 );

                            if ( value == 0.0 )
                                nZeros++;

                            // first run through train data
                            if ( state==0 )
                            {
                                if ( minValues[nF] > value )
                                    minValues[nF] = value;
                                if ( maxValues[nF] < value )
                                    maxValues[nF] = value;

                                // histogram over numeric values
                                int size = numericalAttributes[nF].size();
                                if ( size < numericMaxCluster )
                                {
                                    int foundIndex = -1;
                                    for ( int j=0;j<size;j++ )
                                        if ( numericalAttributes[nF][j] == buf1 )
                                        {
                                            foundIndex = j;
                                            break;
                                        }
                                    // add value
                                    if ( foundIndex == -1 )
                                    {
                                        numericalAttributes[nF].push_back ( buf1 );
                                        numericalAttributesCnt[nF].push_back ( 1 );
                                    }
                                    else
                                        numericalAttributesCnt[nF][foundIndex]++;
                                }

                                if ( value != 0.0 )
                                {
                                    numericNonZeroCnt[nF]++;
                                    if ( numericNonZeroCnt[nF] > nTrain+nTrainTmp+1 )
                                    {
                                        cout<<"numericNonZeroCnt[nF]:"<<numericNonZeroCnt[nF]<<" nF:"<<nF<<" nTrainTmp:"<<nTrainTmp<<" nZeros:"<<nZeros<<" pos0:"<<pos0<<" pos1:"<<pos1<<endl;
                                        assert ( false );
                                    }
                                }

                                if ( value != 0.0 )
                                {
                                    // calc mean over numeric input
                                    meanValues[nF] += value;
                                    mean2Values[nF] += value * value;
                                    meanCnt[nF]++;
                                }
                            }
                            else if ( state==1 ) // second run, fill data tables
                            {
                                if ( numericNonZeroCnt[nF] >= minAttributeOccurenceNumerical && maxNormValues[nF] < stdValues[nF]*maxSTD )
                                {
                                    // numeric add
                                    if ( value == 0.0 && setNumZerosToMeans )
                                        train[nTrainFill*nFeat + nFeatFill] = meanValues[nF];
                                    else
                                        train[nTrainFill*nFeat + nFeatFill] = value;
                                    nFeatFill++;

                                    // numeric one hot add
                                    int size = numericalAttributes[nF].size();
                                    if ( size < numericMaxCluster && size > 1 )
                                    {
                                        int foundIndex = -1;
                                        for ( int j=0;j<size;j++ )
                                            if ( numericalAttributes[nF][j] == buf1 )
                                            {
                                                foundIndex = j;
                                                break;
                                            }
                                        // fill categorical
                                        int beforeHot = nrHot;
                                        for ( int j=0;j<size;j++ )
                                        {
                                            if ( foundIndex == j )
                                            {
                                                train[nTrainFill*nFeat + nFeatFill] = 1.0;
                                                nrHot++;
                                            }
                                            else
                                                train[nTrainFill*nFeat + nFeatFill] = 0.0;
                                            nFeatFill++;
                                        }
                                        // fill missing
                                        /*if(nrHot == beforeHot)
                                            train[nTrainFill*nFeat + nFeatFill] = 0.0;
                                            else
                                            train[nTrainFill*nFeat + nFeatFill] = 1.0;
                                            nFeatFill++;*/
                                    }

                                }

                                // missing values one-hot encoded
                                if ( numericHasMissingBin[nF] )
                                {
                                    train[nTrainFill*nFeat + nFeatFill] = 0.0;  // <- missing
                                    nFeatFill++;
                                    train[nTrainFill*nFeat + nFeatFill] = 1.0;  // <- available
                                    nFeatFill++;
                                }
                            }
                        }
                        //=======================================================================================================
                        //=======================================================================================================
                        // Read Categorical value (feature count >= NUM)
                        //
                        else
                        {
                            int index = nF-NUM;
                            if ( index >= CAT )
                                assert ( false );
                            int size = categoricalAttributes[index].size();
                            int sizeCnt = categoricalAttributesCnt[index].size();
                            if ( size != sizeCnt )
                                assert ( false );

                            int foundIndex = -1;
                            for ( int j=0;j<size;j++ )
                                if ( categoricalAttributes[index][j] == buf1 )
                                {
                                    foundIndex = j;
                                    break;
                                }

                            // first run through train data
                            if ( state==0 )
                            {
                                // add value
                                if ( foundIndex == -1 )
                                {
                                    categoricalAttributes[index].push_back ( buf1 );
                                    categoricalAttributesCnt[index].push_back ( 1 );
                                }
                                else // already exists
                                    categoricalAttributesCnt[index][foundIndex]++;
                            }
                            else if ( state==1 ) // second run, fill data tables
                            {
                                // one-hot encoding
                                int fillCnt = 0;
                                int beforeHot = nrHot;
                                for ( int j=0;j<size;j++ )
                                {
                                    if ( categoricalAttributesCnt[index][j] >= minAttributeOccurenceCategorical && categoricalAttributesCnt[index][j] < nTrain )
                                    {
                                        if ( foundIndex == j )
                                        {
                                            train[nTrainFill*nFeat + nFeatFill] = 1.0;
                                            nrHot++;
                                        }
                                        else
                                            train[nTrainFill*nFeat + nFeatFill] = 0.0;
                                        fillCnt++;
                                        nFeatFill++;
                                    }
                                }

                                // no missing (no consecutive tabs here)
                                if ( categoricalHasMissingBin[index] )
                                {
                                    train[nTrainFill*nFeat + nFeatFill] = 0.0;
                                    fillCnt++;
                                    nFeatFill++;
                                }

                                // if found, but not in cache
                                if ( categoricalHasUnknownBin[index] )
                                {
                                    if ( beforeHot == nrHot )
                                    {
                                        train[nTrainFill*nFeat + nFeatFill] = 1.0;
                                        nrHot++;
                                    }
                                    else
                                        train[nTrainFill*nFeat + nFeatFill] = 0.0;
                                    fillCnt++;
                                    nFeatFill++;
                                }

                                if ( nrHot != beforeHot + 1 && fillCnt > 0 )
                                {
                                    cout<<"WARNING: foundIndex:"<<foundIndex<<" "<<size<<" "<<index<<" "<<nrHot<<" "<<beforeHot<<" fill:"<<fillCnt<<endl;
                                    //assert(false);
                                }
                            }
                        }
                    }
                    //===========================================================================================================
                    //===========================================================================================================
                    // If the feature has no content
                    // Missing value here
                    //
                    else
                    {
                        nMissing++;

                        if ( state==0 )
                        {
                            // numeric
                            if ( nF < NUM )
                            {
                                numericMissingCnt[nF]++;
                            }
                            // categorical
                            if ( nF >= NUM )
                            {
                                int index = nF-NUM;
                                categoricalMissingCnt[index]++;
                            }
                        }

                        // second run, fill data tables with zeros
                        if ( state==1 )
                        {
                            //===================================================================================================
                            //===================================================================================================
                            // Read Numeric value (feature count < NUM)
                            //
                            if ( nF < NUM )
                            {
                                if ( numericNonZeroCnt[nF] >= minAttributeOccurenceNumerical && maxNormValues[nF] < stdValues[nF]*maxSTD )
                                {
                                    // numeric add
                                    if ( setMissingToMeans )
                                        train[nTrainFill*nFeat + nFeatFill] = meanValues[nF];
                                    else
                                        train[nTrainFill*nFeat + nFeatFill] = 0.0;
                                    nFeatFill++;

                                    // numeric one hot add
                                    int size = numericalAttributes[nF].size();
                                    if ( size < numericMaxCluster && size > 1 )
                                    {
                                        // fill categorical
                                        for ( int j=0;j<size;j++ )
                                        {
                                            train[nTrainFill*nFeat + nFeatFill] = 0.0;
                                            nFeatFill++;
                                        }
                                        // fill missing
                                        //train[nTrainFill*nFeat + nFeatFill] = 1.0;
                                        //nFeatFill++;
                                    }
                                }

                                // missing values one-hot encoded
                                if ( numericHasMissingBin[nF] )
                                {
                                    train[nTrainFill*nFeat + nFeatFill] = 1.0;  // <- missing
                                    nFeatFill++;
                                    train[nTrainFill*nFeat + nFeatFill] = 0.0;  // <- available
                                    nFeatFill++;
                                }
                            }
                            //===================================================================================================
                            //===================================================================================================
                            // Read Categorical value (feature count >= NUM)
                            //
                            else
                            {
                                int index = nF - NUM;
                                if ( index >= CAT )
                                    assert ( false );
                                int size = categoricalAttributes[index].size();
                                int sizeCnt = categoricalAttributesCnt[index].size();
                                if ( size != sizeCnt )
                                    assert ( false );

                                // one-hot encoding
                                int fillCnt = 0;
                                int beforeHot = nrHot;
                                for ( int j=0;j<size;j++ )
                                {
                                    if ( categoricalAttributesCnt[index][j] >= minAttributeOccurenceCategorical && categoricalAttributesCnt[index][j] < nTrain )
                                    {
                                        train[nTrainFill*nFeat + nFeatFill] = 0.0;  // no here
                                        fillCnt++;
                                        nFeatFill++;
                                    }
                                }
                                if ( categoricalHasMissingBin[index] )
                                {
                                    if ( fillCnt == 0 && categoricalHasUnknownBin[index] == false )
                                    {
                                        cout<<"categoricalMissingCnt["<<index<<"]:"<<categoricalMissingCnt[index]<<endl;
                                        assert ( false );
                                    }
                                    // set the input to "missing value"
                                    train[nTrainFill*nFeat + nFeatFill] = 1.0;
                                    nrHot++;
                                    fillCnt++;
                                    nFeatFill++;
                                }

                                if ( categoricalHasUnknownBin[index] )
                                {
                                    // no unknown value
                                    train[nTrainFill*nFeat + nFeatFill] = 0.0;
                                    fillCnt++;
                                    nFeatFill++;
                                }

                                if ( nrHot != beforeHot + 1 && fillCnt > 0 )
                                {
                                    cout<<"WARNING: "<<size<<" "<<index<<" "<<nrHot<<" "<<beforeHot<<" fill:"<<fillCnt<<endl;
                                    //assert(false);
                                }
                            }
                        }
                    }

                    // check for last character
                    if ( lineBuf[pos1]!=0 )
                        pos1++;

                    // beginpos = endpos
                    pos0 = pos1;

                    // column count
                    nF++;
                }

                // valid checks
                if ( nF != NUM + CAT )
                    assert ( false );
                if ( state==1 )
                {
                    if ( nFeatFill != nFeat )
                    {
                        cout<<"nFeatFill:"<<nFeatFill<<" nFeat:"<<nFeat<<endl;
                        assert ( false );
                    }
                    nTrainFill++;
                }

                nTrainTmp++;

                sparse += nMissing / ( double ) nF;
                zeroRatio += nZeros / ( double ) nF;
            }

            f.close();

            // ratio of sparseness and zeroPercent
            sparse /= ( double ) nTrainTmp;
            zeroRatio /= ( double ) nTrainTmp;
            cout<<"nTrainTmp:"<<nTrainTmp<<endl;
            cout<<"missing values:"<<100.0*sparse<<"%"<<endl;
            cout<<"zero values:"<<100.0*zeroRatio<<"%"<<endl;

            double min0 = 1e20, max0 = -1e20;
            for ( int i=0;i<100000;i++ )
            {
                if ( min0 > minValues[i] )
                    min0 = minValues[i];
                if ( max0 < maxValues[i] )
                    max0 = maxValues[i];
            }
            cout<<"min|max values: "<<min0<<"|"<<max0<<endl;

            int sum = 0;
            for ( int j=0;j<CAT;j++ )
                sum += categoricalAttributes[j].size();
            cout<<"nCategoricalSum:"<<sum<<endl;

            if ( state == 0 )
                nTrain += nTrainTmp;

        }

        // do some checks
        if ( state == 1 )
        {
            if ( nTrain != nTrainFill )
                assert ( false );

            for ( int i=0;i<nTrain*nFeat;i++ )
                if ( train[i] == 1e10 )
                {
                    cout<<"i:"<<i<<endl;
                    assert ( false );
                }
        }

        if ( state==0 )
        {
            for ( int i=0;i<NUM;i++ )
                numericNonZeroPercent[i] = ( double ) numericNonZeroCnt[i]/ ( double ) nTrain;
            for ( int i=0;i<100000;i++ )
                if ( meanCnt[i] > 0 )
                {
                    meanValues[i] /= ( double ) meanCnt[i];
                    stdValues[i] = sqrt ( mean2Values[i]/ ( double ) meanCnt[i] - meanValues[i]/ ( double ) meanCnt[i] );
                    maxNormValues[i] = fabs ( maxValues[i] - meanValues[i] );
                    if ( maxNormValues[i] < fabs ( minValues[i] - meanValues[i] ) )
                        maxNormValues[i] = fabs ( minValues[i] - meanValues[i] );
                }

            cout<<"nTrain:"<<nTrain<<endl;

            // === Calculate effective number of input features ===
            nFeat = 1; // const
            int nFeatNum = 0, nFeatNumRaw = 0, nFeatNumCat = 0, nFeatCat = 0, nUnknown = 0, nMissing = 0, nIn = 0, nNumMiss = 0;
            // numerical
            for ( int j=0;j<NUM;j++ )
            {
                if ( numericNonZeroCnt[j] >= minAttributeOccurenceNumerical && maxNormValues[j] < stdValues[j]*maxSTD )
                {
                    // standard numerical input
                    nFeat++;
                    nFeatNum++;
                    nFeatNumRaw++;

                    // numerical input with limited number of different values -> translate it to categorical input
                    if ( numericalAttributes[j].size() < numericMaxCluster && numericalAttributes[j].size() > 1 )
                    {
                        cout<<"nFeatNum:"<<nFeatNum<<" ";
                        for ( int k=0;k<numericalAttributes[j].size();k++ )
                        {
                            cout<<numericalAttributes[j][k]<<"("<<numericalAttributesCnt[j][k]<<") ";
                            nFeat++;
                            nFeatNum++;
                            nFeatNumCat++;
                        }
                        cout<<endl;
                        /*
                        // add one bin for "missing or unknown value"
                            nFeat++;
                            nFeatNum++;
                            nFeatNumCat++;*/
                    }
                    if ( numericMissingCnt[j] >= numericMinMissing )
                    {
                        numericHasMissingBin[j] = true;
                        nFeat+=2;
                        nNumMiss+=2;
                    }
                }
            }
            // categorical
            for ( int j=0;j<CAT;j++ )
            {
                int nUsed = 0, nUn = 0, nCat = 0, nMiss = 0, nUnk = 0;
                for ( int k=0;k<categoricalAttributesCnt[j].size();k++ )
                {
                    // count valid entries (with enough occurence)
                    if ( categoricalAttributesCnt[j][k] >= minAttributeOccurenceCategorical && categoricalAttributesCnt[j][k] < nTrain )
                    {
                        nFeat++;
                        nFeatCat++;
                        nUsed++;
                        nIn++;
                        nCat++;
                    }
                    else if ( categoricalAttributesCnt[j][k] < nTrain ) // not enough occurence -> put to unknown
                        nUn++;
                }
                // missing is like a normal categoric input
                if ( ( categoricalMissingCnt[j] >= minAttributeOccurenceCategorical && categoricalMissingCnt[j] < nTrain ) || categoricalMissingCnt[j] > 0 && nCat > 0 )
                {
                    // add a "missing value" input of this feature
                    nFeat++;
                    nFeatCat++;
                    nMissing++;
                    nMiss++;
                    categoricalHasMissingBin[j] = true;
                }
                if ( nUn > 0 && nCat + nMiss > 0 )
                {
                    // add a "unknown value" input of this feature
                    nFeat++;
                    nFeatCat++;
                    nUnknown++;
                    nUnk++;
                    categoricalHasUnknownBin[j] = true;
                }

                if ( nCat + nMiss + nUnk == 1 )
                    assert ( false );
            }

            cout<<"nFeat:"<<nFeat<<" (numInputs:"<<nFeatNum<<" [rawNum:"<<nFeatNumRaw<<" nFeatNumCat:"<<nFeatNumCat<<"] catInputs:"<<nFeatCat<<" [nUnknown:"<<nUnknown<<" nMissing:"<<nMissing<<" nCat:"<<nIn<<"] numMissingHot:"<<nNumMiss<<" [+1const.])"<<endl;

            cout<<"Allocate train features: "<< ( double ) nTrain*nFeat/1e6*4.0<<" MB"<<endl;
            train = new REAL[nTrain*nFeat];
            for ( int i=0;i<nTrain*nFeat;i++ )
                train[i] = 1e10;

            //support = new REAL[nFeat];
            //supportCnt = new int[nFeat];
            //for(int i=0;i<nFeat;i++)
            //{
            //support[i] = 0.0;
            //supportCnt[i] = 0;
            //}

            // read targets
            nClass = 2;
            trainTarget = new REAL[nTrain*nClass*nDomain];
            trainLabel = new int[nTrain*nDomain];
            for ( int d=0;d<nDomain;d++ )
            {
                sprintf ( buf0,"%s/%s",path.c_str(),targetFiles[d] );
                fstream f;
                cout<<"Open targets:"<<buf0<<endl;
                f.open ( buf0,ios::in );
                if ( f.is_open() == false )
                    assert ( false );
                int label;
                for ( int i=0;i<nTrain;i++ )
                {
                    f>>label;
                    if ( label==-1 )
                    {
                        trainTarget[i*nClass*nDomain + d*nClass + 0] = positiveTarget;
                        trainTarget[i*nClass*nDomain + d*nClass + 1] = negativeTarget;
                        trainLabel[i*nDomain + d] = 0;
                    }
                    else if ( label==1 )
                    {
                        trainTarget[i*nClass*nDomain + d*nClass + 0] = negativeTarget;
                        trainTarget[i*nClass*nDomain + d*nClass + 1] = positiveTarget;
                        trainLabel[i*nDomain + d] = 1;
                    }
                    else
                        assert ( false );
                }
                f.close();
            }
            // test set
            nTest = 0;
            test = 0;
            testTarget = 0;
            testLabel = 0;

        }
    }

    for ( int i=0;i<nTrain;i++ )
        for ( int j=0;j<nFeat;j++ )
            if ( train[i*nFeat+j] == 1e10 )
            {
                cout<<"i:"<<i<<" j:"<<j<<" "<<train[i*nFeat+j]<<endl;
                assert ( false );
            }


    fstream f;
    if ( lineBuf )
    {
        delete[] lineBuf;
        lineBuf = 0;
    }
    if ( numericNonZeroCnt )
    {
        delete[] numericNonZeroCnt;
        numericNonZeroCnt = 0;
    }
    if ( numericNonZeroPercent )
    {
        delete[] numericNonZeroPercent;
        numericNonZeroPercent = 0;
    }
    if ( categoricalAttributes )
    {
        delete[] categoricalAttributes;
        categoricalAttributes = 0;
    }
    if ( meanValues )
    {
        delete[] meanValues;
        meanValues = 0;
    }
    if ( meanCnt )
    {
        delete[] meanCnt;
        meanCnt = 0;
    }
    if ( categoricalHasMissingBin )
    {
        delete[] categoricalHasMissingBin;
        categoricalHasMissingBin = 0;
    }

    if ( Framework::getFrameworkMode() == 1 )
    {
        cout<<endl<<"Set test data (and set train data to 0)"<<endl<<endl;
        test = train;
        train = 0;
        nTest = nTrain;
        nTrain = 0;
        testTarget = trainTarget;
        trainTarget = 0;
        testLabel = trainLabel;
        trainLabel = 0;
    }

    cout<<endl<<"Finished read in "<<time ( 0 )-t0<<"[s]"<<endl<<endl;

}

void DatasetReader::readLETTER	(	string	path,
		REAL *&	train,
		REAL *&	trainTarget,
		int *&	trainLabel,
		REAL *&	test,
		REAL *&	testTarget,
		int *&	testLabel,
		uint &	nTrain,
		uint &	nTest,
		int &	nClass,
		int &	nDomain,
		int &	nFeat,
		REAL	positiveTarget = 1.0,
		REAL	negativeTarget = -1.0
	)

Reads the LETTER dataset (UCI)

712565Bytes letter-recognition.data 2734Bytes letter-recognition.names

Definition at line 3843 of file DatasetReader.cpp.

{
    cout<<"Read LETTER from: "<<path<<endl;
    nDomain = 1;

    // define data type and files
    int targetColumn = 1;
    uint nTrainTmp;
    char columnType[] = "dnnnnnnnnnnnnnnnn";
    char enabledCol[] = "11111111111111111";
    const char* dataFiles[] = { ( new string ( path+"/letter-recognition.data" ) )->c_str(),0};

    // === TRAIN SET ===
    getDataBounds ( dataFiles, ",", nFeat, nClass, nTrainTmp, columnType, enabledCol, targetColumn, 0 );

    // allocate tmp mem
    REAL* trainTmp = new REAL[nTrainTmp * nFeat];
    int* trainLabelTmp = new int[nTrainTmp];

    // fill data
    getDataBounds ( dataFiles, ",", nFeat, nClass, nTrainTmp, columnType, enabledCol, targetColumn, 0, true, trainTmp, trainLabelTmp );

    // split train and testset from trainTmp
    splitRandomTestset ( 0.2, trainTmp, trainLabelTmp, nTrainTmp, nFeat, nClass, train, trainLabel, trainTarget, test, testLabel, testTarget, nTrain, nTest, positiveTarget, negativeTarget, true );  // true = take the last n percent (without random selection)

    delete[] trainTmp;
    delete[] trainLabelTmp;

}

void DatasetReader::readMNIST	(	string	path,
		REAL *&	train,
		REAL *&	trainTarget,
		int *&	trainLabel,
		REAL *&	test,
		REAL *&	testTarget,
		int *&	testLabel,
		uint &	nTrain,
		uint &	nTest,
		int &	nClass,
		int &	nDomain,
		int &	nFeat,
		REAL	positiveTarget = 1.0,
		REAL	negativeTarget = -1.0
	)

Reads the dataset from following files Trainset: train-images-idx3-ubyte (47040016 Bytes), train-labels-idx1-ubyte (60008 Bytes) Testset: t10k-images-idx3-ubyte (7840016 Bytes), t10k-labels-idx1-ubyte (10008 Bytes)

Definition at line 2365 of file DatasetReader.cpp.

{
    cout<<"Read MNIST from: "<<path<<endl;

    fstream fTrain ( ( path+"/"+string ( "train-images-idx3-ubyte" ) ).c_str(), ios::in );
    fstream fTrainLabels ( ( path+"/"+string ( "train-labels-idx1-ubyte" ) ).c_str(), ios::in );
    fstream fTest ( ( path+"/"+string ( "t10k-images-idx3-ubyte" ) ).c_str(), ios::in );
    fstream fTestLabels ( ( path+"/"+string ( "t10k-labels-idx1-ubyte" ) ).c_str(), ios::in );

    if ( fTrain.is_open() ==false || fTrainLabels.is_open() ==false || fTest.is_open() ==false || fTestLabels.is_open() ==false )
    {
        cout<<"Error in opening the files"<<endl;
        exit ( 0 );
    }

    // population
    nClass = 10;
    nDomain = 1;
    nTrain = 60000;
    nTest  = 10000;
    nFeat = 784;    // (28 x 28 pixel 8-Bit images)

    // allocate mem
    unsigned char* trainChar = new unsigned char[nTrain * nFeat];
    unsigned char* testChar = new unsigned char[nTest * nFeat];
    unsigned char* trainLabelChar = new unsigned char[nTrain];
    unsigned char* testLabelChar = new unsigned char[nTest];

    // load raw data
    unsigned int dummy;
    fTrain.read ( ( char* ) &dummy, sizeof ( int ) );  // magic number
    fTrain.read ( ( char* ) &dummy, sizeof ( int ) );  // #images
    fTrain.read ( ( char* ) &dummy, sizeof ( int ) );  // rows
    fTrain.read ( ( char* ) &dummy, sizeof ( int ) );  // cols
    fTrain.read ( ( char* ) trainChar, sizeof ( unsigned char ) *nTrain*nFeat );  // images
    fTrain.close();

    fTrainLabels.read ( ( char* ) &dummy, sizeof ( int ) );  // magic number
    fTrainLabels.read ( ( char* ) &dummy, sizeof ( int ) );  // #items
    fTrainLabels.read ( ( char* ) trainLabelChar, sizeof ( unsigned char ) *nTrain );  // labels
    fTrainLabels.close();

    fTest.read ( ( char* ) &dummy, sizeof ( int ) );  // magic number
    fTest.read ( ( char* ) &dummy, sizeof ( int ) );  // #images
    fTest.read ( ( char* ) &dummy, sizeof ( int ) );  // rows
    fTest.read ( ( char* ) &dummy, sizeof ( int ) );  // cols
    fTest.read ( ( char* ) testChar, sizeof ( unsigned char ) *nTest*nFeat );  // images
    fTest.close();

    fTestLabels.read ( ( char* ) &dummy, sizeof ( int ) );  // magic number
    fTestLabels.read ( ( char* ) &dummy, sizeof ( int ) );  // #items
    fTestLabels.read ( ( char* ) testLabelChar, sizeof ( unsigned char ) *nTest );  // labels
    fTestLabels.close();

    // row x col train images as test pgm file
    int rows = 50, cols = 100;
    fstream fimg ( ( path + "/MNIST.pgm" ).c_str(),ios::out );
    char buf[256];
    sprintf ( buf,"P5\n%d %d\n255\n", cols*28, rows*28 );
    fimg<<buf;
    // image
    for ( int I=0;I<rows;I++ )
    {
        // write image
        for ( int j=0;j<28;j++ )
        {
            for ( int i=0;i<cols;i++ )
            {
                for ( int k=0;k<28;k++ )
                    fimg.write ( ( char* ) &trainChar[k + i*nFeat + j*28 + I*cols*nFeat], sizeof ( unsigned char ) );
            }
        }
    }
    fimg.close();

    // allocate + write dataset
    train = new REAL[nTrain * nFeat];
    trainLabel = new int[nTrain];
    test = new REAL[nTest * nFeat];
    testLabel = new int[nTest];

    for ( int i=0;i<nTrain;i++ )
    {
        trainLabel[i] = ( int ) trainLabelChar[i];
        for ( int j=0;j<nFeat;j++ )
            train[i*nFeat + j] = ( REAL ) trainChar[i*nFeat + j] / 255.0;
    }

    for ( int i=0;i<nTest;i++ )
    {
        testLabel[i] = ( int ) testLabelChar[i];
        for ( int j=0;j<nFeat;j++ )
            test[i*nFeat + j] = ( REAL ) testChar[i*nFeat + j] / 255.0;
    }

    // train targets
    trainTarget = new REAL[nClass*nTrain];
    for ( int i=0;i<nTrain;i++ )
    {
        for ( int j=0;j<nClass;j++ )
            trainTarget[i*nClass + j] = negativeTarget;  // negative class labels
        trainTarget[i*nClass + trainLabel[i]] = positiveTarget;  // positive class label
    }

    // test targets
    testTarget = new REAL[nClass*nTest];
    for ( int i=0;i<nTest;i++ )
    {
        for ( int j=0;j<nClass;j++ )
            testTarget[i*nClass + j] = negativeTarget;  // negative class labels
        testTarget[i*nClass + testLabel[i]] = positiveTarget;  // positive class label
    }

    // free raw data
    if ( trainChar )
    {
        delete[] trainChar;
        trainChar = 0;
    }
    if ( testChar )
    {
        delete[] testChar;
        testChar = 0;
    }
    if ( trainLabelChar )
    {
        delete[] trainLabelChar;
        trainLabelChar = 0;
    }
    if ( testLabelChar )
    {
        delete[] testLabelChar;
        testLabelChar = 0;
    }
}

void DatasetReader::readMONKS1	(	string	path,
		REAL *&	train,
		REAL *&	trainTarget,
		int *&	trainLabel,
		REAL *&	test,
		REAL *&	testTarget,
		int *&	testLabel,
		uint &	nTrain,
		uint &	nTest,
		int &	nClass,
		int &	nDomain,
		int &	nFeat,
		REAL	positiveTarget = 1.0,
		REAL	negativeTarget = -1.0
	)

Reads the MONKS1 dataset (UCI)

10CATBytes monks-1.test 2947Bytes monks-1.train

Definition at line 3879 of file DatasetReader.cpp.

{
    cout<<"Read MONKS1 from: "<<path<<endl;
    nDomain = 1;

    // define data type and files
    int targetColumn = 1;
    char columnType[] = "dnnnnnnd";
    char enabledCol[] = "11111110";
    const char* dataFiles[] = { ( new string ( path+"/monks-1.train" ) )->c_str(), ( new string ( path+"/monks-1.test" ) )->c_str(),0};

    // === TRAIN SET ===
    getDataBounds ( dataFiles, " ", nFeat, nClass, nTrain, columnType, enabledCol, targetColumn, 0 );
    train = new REAL[nFeat*nTrain];
    trainLabel = new int[nTrain];
    getDataBounds ( dataFiles, " ", nFeat, nClass, nTrain, columnType, enabledCol, targetColumn, 0, true, train, trainLabel );

    // === TEST SET ===
    getDataBounds ( dataFiles, " ", nFeat, nClass, nTest, columnType, enabledCol, targetColumn, 1 );
    test = new REAL[nFeat*nTest];
    testLabel = new int[nTest];
    getDataBounds ( dataFiles, " ", nFeat, nClass, nTest, columnType, enabledCol, targetColumn, 1, true, test, testLabel );

    // make numerical test targets
    makeNumericTrainAndTestTargets ( nClass, nTrain, nTest, positiveTarget, negativeTarget, trainLabel, testLabel, trainTarget, testTarget );

}

void DatasetReader::readMONKS2	(	string	path,
		REAL *&	train,
		REAL *&	trainTarget,
		int *&	trainLabel,
		REAL *&	test,
		REAL *&	testTarget,
		int *&	testLabel,
		uint &	nTrain,
		uint &	nTest,
		int &	nClass,
		int &	nDomain,
		int &	nFeat,
		REAL	positiveTarget = 1.0,
		REAL	negativeTarget = -1.0
	)

Reads the MONKS2 dataset (UCI)

10CATBytes monks-2.test 4013Bytes monks-2.train

Definition at line 3913 of file DatasetReader.cpp.

{
    cout<<"Read MONKS2 from: "<<path<<endl;
    nDomain = 1;

    // define data type and files
    int targetColumn = 1;
    char columnType[] = "dnnnnnnd";
    char enabledCol[] = "11111110";
    const char* dataFiles[] = { ( new string ( path+"/monks-2.train" ) )->c_str(), ( new string ( path+"/monks-2.test" ) )->c_str(),0};

    // === TRAIN SET ===
    getDataBounds ( dataFiles, " ", nFeat, nClass, nTrain, columnType, enabledCol, targetColumn, 0 );
    train = new REAL[nFeat*nTrain];
    trainLabel = new int[nTrain];
    getDataBounds ( dataFiles, " ", nFeat, nClass, nTrain, columnType, enabledCol, targetColumn, 0, true, train, trainLabel );

    // === TEST SET ===
    getDataBounds ( dataFiles, " ", nFeat, nClass, nTest, columnType, enabledCol, targetColumn, 1 );
    test = new REAL[nFeat*nTest];
    testLabel = new int[nTest];
    getDataBounds ( dataFiles, " ", nFeat, nClass, nTest, columnType, enabledCol, targetColumn, 1, true, test, testLabel );

    // make numerical test targets
    makeNumericTrainAndTestTargets ( nClass, nTrain, nTest, positiveTarget, negativeTarget, trainLabel, testLabel, trainTarget, testTarget );

}

void DatasetReader::readMONKS3	(	string	path,
		REAL *&	train,
		REAL *&	trainTarget,
		int *&	trainLabel,
		REAL *&	test,
		REAL *&	testTarget,
		int *&	testLabel,
		uint &	nTrain,
		uint &	nTest,
		int &	nClass,
		int &	nDomain,
		int &	nFeat,
		REAL	positiveTarget = 1.0,
		REAL	negativeTarget = -1.0
	)

Reads the MONKS3 dataset (UCI)

10CATBytes monks-3.test 2886Bytes monks-3.train

Definition at line 3947 of file DatasetReader.cpp.

{
    cout<<"Read MONKS3 from: "<<path<<endl;
    nDomain = 1;

    // define data type and files
    int targetColumn = 1;
    char columnType[] = "dnnnnnnd";
    char enabledCol[] = "11111110";
    const char* dataFiles[] = { ( new string ( path+"/monks-3.train" ) )->c_str(), ( new string ( path+"/monks-3.test" ) )->c_str(),0};

    // === TRAIN SET ===
    getDataBounds ( dataFiles, " ", nFeat, nClass, nTrain, columnType, enabledCol, targetColumn, 0 );
    train = new REAL[nFeat*nTrain];
    trainLabel = new int[nTrain];
    getDataBounds ( dataFiles, " ", nFeat, nClass, nTrain, columnType, enabledCol, targetColumn, 0, true, train, trainLabel );

    // === TEST SET ===
    getDataBounds ( dataFiles, " ", nFeat, nClass, nTest, columnType, enabledCol, targetColumn, 1 );
    test = new REAL[nFeat*nTest];
    testLabel = new int[nTest];
    getDataBounds ( dataFiles, " ", nFeat, nClass, nTest, columnType, enabledCol, targetColumn, 1, true, test, testLabel );

    // make numerical test targets
    makeNumericTrainAndTestTargets ( nClass, nTrain, nTest, positiveTarget, negativeTarget, trainLabel, testLabel, trainTarget, testTarget );

}

void DatasetReader::readMUSHROOM	(	string	path,
		REAL *&	train,
		REAL *&	trainTarget,
		int *&	trainLabel,
		REAL *&	test,
		REAL *&	testTarget,
		int *&	testLabel,
		uint &	nTrain,
		uint &	nTest,
		int &	nClass,
		int &	nDomain,
		int &	nFeat,
		REAL	positiveTarget = 1.0,
		REAL	negativeTarget = -1.0
	)

Reads the MUSHROOM dataset (UCI)

373704Bytes agaricus-lepiota.data 6816Bytes agaricus-lepiota.names

Definition at line 3981 of file DatasetReader.cpp.

{
    cout<<"Read MUSHROOM from: "<<path<<endl;
    nDomain = 1;

    // define data type and files
    int targetColumn = 1;
    uint nTrainTmp;
    char columnType[] = "ddddddddddddddddddddddd";
    char enabledCol[] = "11111111111111111111111";
    const char* dataFiles[] = { ( new string ( path+"/agaricus-lepiota.data" ) )->c_str(),0};

    // === TRAIN SET ===
    getDataBounds ( dataFiles, ",", nFeat, nClass, nTrainTmp, columnType, enabledCol, targetColumn, 0 );

    // allocate tmp mem
    REAL* trainTmp = new REAL[nTrainTmp * nFeat];
    int* trainLabelTmp = new int[nTrainTmp];

    // fill data
    getDataBounds ( dataFiles, ",", nFeat, nClass, nTrainTmp, columnType, enabledCol, targetColumn, 0, true, trainTmp, trainLabelTmp );

    // split train and testset from trainTmp
    splitRandomTestset ( 0.2, trainTmp, trainLabelTmp, nTrainTmp, nFeat, nClass, train, trainLabel, trainTarget, test, testLabel, testTarget, nTrain, nTest, positiveTarget, negativeTarget );

    delete[] trainTmp;
    delete[] trainLabelTmp;

}

void DatasetReader::readNETFLIX	(	string	path,
		REAL *&	train,
		REAL *&	trainTarget,
		int *&	trainLabel,
		REAL *&	test,
		REAL *&	testTarget,
		int *&	testLabel,
		uint &	nTrain,
		uint &	nTest,
		int &	nClass,
		int &	nDomain,
		int &	nFeat,
		REAL	positiveTarget = 1.0,
		REAL	negativeTarget = -1.0
	)

Reads the binary NETFLIX prediction files in the DataFiles folder e.g. prediction.dat: 16902104 Bytes, linear float precision [probe,qualifying] probeRatings.txt: 4
... 1408395 ratings (1..5)

Definition at line 2775 of file DatasetReader.cpp.

{
    cout<<"Read NETFLIX binary predictions from: "<<NETFLIX_DATA_DIR<<endl;

    if ( Framework::getAdditionalStartupParameter() < 0 )
    {
        // probeset subsampling
        if ( Framework::getAdditionalStartupParameter() < -100 )
        {
            srand ( Framework::getRandomSeed() );

            cout<<"Probeset subsampling"<<endl;

            // population
            nClass = 1;   // -> one regression target
            nDomain = 1;
            nTrain = 1408395;  // #probe
            nTest = 2817131;   // #qual

            bool* maskProbe = new bool[nTrain];
            for ( int i=0;i<nTrain;i++ )
                maskProbe[i] = false;
            double p = - ( double ) ( Framework::getAdditionalStartupParameter() ) / ( double ) nTrain;
            int c = 0;
            for ( int i=0;i<nTrain;i++ )
                if ( ( double ) rand() / ( double ) RAND_MAX < p )
                {
                    maskProbe[i] = true;
                    c++;
                }
            cout<<"Selected: "<<c<<" probe samples"<<endl;


            // get all the data files
            vector<string> files = Data::getDirectoryFileList ( NETFLIX_DATA_DIR );
            vector<string> predictionFiles;

            // read the *.dat files (prediction of probe+qual files)
            nFeat = 0;
            for ( int i=0;i<files.size();i++ )
            {
                int pos = files[i].find ( ".dat" );
                string fileEnding = files[i].substr ( files[i].length()-4,4 );
                if ( fileEnding == ".dat" )
                {
                    predictionFiles.push_back ( files[i] );
                    nFeat++;
                }
            }

            cout<<"nFeat: "<<nFeat<<endl;
            cout<<"nClass: "<<nClass<<endl;

            // probe targets
            //cout<<"Targets Read:"<<path+"/"+string("probeRatings.txt")<<endl;
            //fstream fProbeRatings((path+"/"+string("probeRatings.txt")).c_str(), ios::in);
            cout<<"Targets Read:"<<path+"/"+string ( "probeRatings.txt.rand" ) <<endl;
            fstream fProbeRatings ( ( path+"/"+string ( "probeRatings.txt.rand" ) ).c_str(), ios::in );
            float* ratingCache = new float[nTrain];
            for ( int i=0;i<nTrain;i++ )
                fProbeRatings>>ratingCache[i];
            fProbeRatings.close();


            test = 0;
            testLabel = 0;
            testTarget = 0;
            if ( Framework::getFrameworkMode() == 1 )
            {
                test = new REAL[ ( nTrain+nTest ) * nFeat];
                testTarget = new REAL[nTrain+nTest];
                for ( int i=0;i<nTrain+nTest;i++ )
                    testTarget[i] = 0.0;
            }
            train = new REAL[c * nFeat];
            trainLabel = 0;
            trainTarget = new REAL[c];
            int d = 0;
            for ( int j=0;j<nTrain;j++ )
            {
                if ( maskProbe[j] )
                {
                    trainTarget[d] = ratingCache[j];
                    d++;
                }
            }

            // predictions
            float* trainTmp = new float[nTrain+nTest];
            for ( int i=0;i<predictionFiles.size();i++ )
            {
                fstream f ( predictionFiles[i].c_str(), ios::in );
                f.read ( ( char* ) trainTmp, sizeof ( float ) * ( nTrain+nTest ) );
                if ( Framework::getFrameworkMode() == 1 )
                    for ( int j=0;j<nTrain+nTest;j++ )
                        test[j*nFeat + i] = trainTmp[j];
                d = 0;
                for ( int j=0;j<nTrain;j++ )
                {
                    if ( maskProbe[j] )
                    {
                        train[d*nFeat + i] = trainTmp[j];
                        d++;
                    }
                }
                f.close();
                cout<<"Prediction file: "<<predictionFiles[i]<<endl;
            }

            delete[] trainTmp;
            delete[] ratingCache;
            delete[] maskProbe;

            nTest = nTrain + nTest;
            nTrain = c;
            cout<<"nTrain:"<<nTrain<<endl<<"nTest:"<<nTest<<endl<<endl;
            return;
        }


        // population
        nClass = 1;   // -> one regression target
        nDomain = 1;
        //nTrain = 1408395;  // #probe
        //nTest = 2817131;   // #qual

        // HACK: divide probe into 2 halfs
        nTrain = 704197;  // #probe
        nTest = 704198;   // #qual

        // get all the data files
        cout<<"read path from:"<<path+"/path.txt"<<endl;
        fstream fP ( ( path+"/path.txt" ).c_str(),ios::in );
        string predictorPath;
        fP>>predictorPath;
        cout<<"path:"<<predictorPath<<endl;
        fP.close();
        
        //vector<string> files = Data::getDirectoryFileList(NETFLIX_DATA_DIR);
        vector<string> files = Data::getDirectoryFileList ( predictorPath );
        sort(files.begin(), files.end());
        vector<string> predictionFiles;

        // read the *.dat files (prediction of probe+qual files)
        nFeat = 0;
        for ( int i=0;i<files.size();i++ )
        {
            int pos = files[i].find ( ".dat" );
            string fileEnding = files[i].substr ( files[i].length()-4,4 );
            if ( fileEnding == ".dat" )
            {
                predictionFiles.push_back ( files[i] );
                nFeat++;
            }
        }

        // =============== write the qual parts ================
        int nProbe = 1408395;
        int nQual = 2817131;
        REAL* tmp = new float[nProbe+nQual];
        REAL* tmp2 = new float[predictionFiles.size()*nQual];
        int* tmp3 = new int[nQual];
        fstream ff((predictorPath+"/grand_prize/judging.txt").c_str(),ios::in);
        char buf[1024];
        int cnt = 0;
        while(ff.getline(buf,1024))  // read judging.txt
        {
            string line(buf);
            if(line.length() > 0)
            {
                if(line[line.length()-2] != ':')
                {
                    int nr = atoi(line.c_str());
                    tmp3[cnt] = nr;
                    cnt++;
                }
            }
        }
        assert(cnt==nQual);
        ff.close();
        for ( int i=0;i<predictionFiles.size();i++ )
        {
            fstream f ( predictionFiles[i].c_str(), ios::in );
            f.read ( ( char* ) tmp, sizeof ( float ) *(nProbe+nQual) );
            for(int j=0;j<nQual;j++)
                tmp2[j*predictionFiles.size()+i] = tmp[nProbe+j];
            f.close();
        }
        fstream trainCSV((path+"/testQual.csv").c_str(), ios::out);  // write CSV file
        for(int i=0;i<nQual;i++)
        {
            for(int j=0;j<predictionFiles.size();j++)
                trainCSV<<tmp2[i*nFeat+j]<<",";
            trainCSV<<tmp3[i]<<endl;
        }
        trainCSV.close();
        exit(0);
        // =============== write the qual parts ================
        
        
        
        cout<<"nFeat: "<<nFeat<<endl;
        cout<<"nClass: "<<nClass<<endl;

        bool doClipping = true;
        if ( Framework::getAdditionalStartupParameter() == -2 )
            doClipping = false;

        // allocate complete dataset
        if ( Framework::getFrameworkMode() == 0 )
        {
            train = new REAL[nTrain * nFeat];
            trainLabel = 0; //new int[nTrain];
            trainTarget = new REAL[nTrain * nClass];

            // probe targets
            //cout<<"Targets Read:"<<path+"/"+string("probeRatings.txt")<<endl;
            //fstream fProbeRatings((path+"/"+string("probeRatings.txt")).c_str(), ios::in);
            cout<<"Targets Read:"<<path+"/"+string ( "probeRatings.txt.rand" ) <<endl;
            fstream fProbeRatings ( ( path+"/"+string ( "probeRatings.txt.rand" ) ).c_str(), ios::in );
            for ( int i=0;i<nTrain;i++ )
                fProbeRatings>>trainTarget[i];
            fProbeRatings.close();

            float* trainTmp = new float[nTrain];

            // predictions
            for ( int i=0;i<predictionFiles.size();i++ )
            {
                fstream f ( predictionFiles[i].c_str(), ios::in );
                f.read ( ( char* ) trainTmp, sizeof ( float ) *nTrain );
                double mean = 0.0;
                for ( int j=0;j<nTrain;j++ )
                    mean += trainTmp[j];
                mean /= ( double ) nTrain;
                if ( mean > 1.0 && mean < 5.0 && doClipping )
                    cout<<"[clip] ";
                for ( int j=0;j<nTrain;j++ )
                {
                    train[j*nFeat + i] = trainTmp[j];
                    if ( mean > 1.0 && mean < 5.0 && doClipping )
                    {
                        if ( train[j*nFeat + i] > 5.0 )
                            train[j*nFeat + i] = 5.0;
                        if ( train[j*nFeat + i] < 1.0 )
                            train[j*nFeat + i] = 1.0;
                    }
                }
                f.close();
                cout<<"Prediction file: "<<predictionFiles[i]<<" mean:"<<mean<<endl;
            }

            if ( trainTmp )
            {
                delete[] trainTmp;
                trainTmp = 0;
            }

            test = 0;
            testLabel = 0;
            testTarget = 0;
            nTest = 0;
            
            // write CSV file
            /*fstream trainCSV((path+"/train.csv").c_str(), ios::out);
            for(int i=0;i<nTrain;i++)
            {
                for(int j=0;j<nFeat;j++)
                    trainCSV<<train[i*nFeat+j]<<",";
                trainCSV<<trainTarget[i]<<endl;
            }
            trainCSV.close();*/
        }
        
        if ( Framework::getFrameworkMode() == 1 )
        {
            cout<<"alloc: "<<nTest * ( uint ) nFeat<<endl;
            test = new REAL[nTest * ( uint ) nFeat];
            testLabel = 0; //new int[nTest];
            testTarget = new REAL[nTest * ( uint ) nClass];

            // dummy targets
            for ( int i=0;i<nTest;i++ )
                testTarget[i] = 3.7;  // just a init value (not known in netflix prize)

            // HACK: read 2nd half of probe, this act as a test set
            cout<<"Targets Read:"<<path+"/"+string ( "probeRatings.txt.rand" ) <<endl;
            fstream fProbeRatings ( ( path+"/"+string ( "probeRatings.txt.rand" ) ).c_str(), ios::in );
            REAL dummy;
            for ( int i=0;i<nTrain;i++ )
                fProbeRatings>>dummy;
            for ( int i=0;i<nTest;i++ )
                fProbeRatings>>testTarget[i];
            fProbeRatings.close();

            float* testTmp = new float[nTest];

            // predictions
            for ( uint i=0;i<predictionFiles.size();i++ )
            {
                fstream f ( predictionFiles[i].c_str(), ios::in );
                f.read ( ( char* ) testTmp, sizeof ( float ) *nTrain );  // probe read (dummy)
                f.read ( ( char* ) testTmp, sizeof ( float ) *nTest );
                double mean = 0.0;
                for ( int j=0;j<nTest;j++ )
                    mean += testTmp[j];
                mean /= ( double ) nTest;
                if ( mean > 1.0 && mean < 5.0 && doClipping )
                    cout<<"[clip] ";
                for ( uint j=0;j<nTest;j++ )
                {
                    test[j* ( uint ) nFeat + i] = testTmp[j];
                    if ( mean > 1.0 && mean < 5.0 && doClipping )
                    {
                        if ( test[j* ( uint ) nFeat + i] > 5.0 )
                            test[j* ( uint ) nFeat + i] = 5.0;
                        if ( test[j* ( uint ) nFeat + i] < 1.0 )
                            test[j* ( uint ) nFeat + i] = 1.0;
                    }
                }
                f.close();
                cout<<"Prediction file: "<<predictionFiles[i]<<" mean:"<<mean<<endl;
            }

            if ( testTmp )
            {
                delete[] testTmp;
                testTmp = 0;
            }

            train = 0;
            trainLabel = 0;
            trainTarget = 0;
            nTrain = 0;

            // write CSV file
            /*fstream testCSV((path+"/test.csv").c_str(), ios::out);
            for(int i=0;i<nTest;i++)
            {
                for(int j=0;j<nFeat;j++)
                    testCSV<<test[i*nFeat+j]<<",";
                testCSV<<testTarget[i]<<endl;
            }
            testCSV.close();*/
        }
    }
    else // slot blend
    {
        // population
        nClass = 1;   // -> one regression target
        nDomain = 1;
        char buf0[512];
        char buf1[512];
        char buf2[512];
        char buf3[512];
        char buf4[512];
        sprintf ( buf0,"%s/%s%d/",string ( NETFLIX_SLOTDATA_ROOT_DIR ).c_str(),"p",Framework::getAdditionalStartupParameter() );
        sprintf ( buf1,"%s/%s%d/nProbe.data",string ( NETFLIX_SLOTDATA_ROOT_DIR ).c_str(),"p",Framework::getAdditionalStartupParameter() );
        sprintf ( buf2,"%s/%s%d/nQual.data",string ( NETFLIX_SLOTDATA_ROOT_DIR ).c_str(),"p",Framework::getAdditionalStartupParameter() );
        sprintf ( buf3,"%s/%s%d/ratings.data",string ( NETFLIX_SLOTDATA_ROOT_DIR ).c_str(),"p",Framework::getAdditionalStartupParameter() );
        sprintf ( buf4,"%s/%s%d/ratingsTest.data",string ( NETFLIX_SLOTDATA_ROOT_DIR ).c_str(),"p",Framework::getAdditionalStartupParameter() );

        fstream f;

        // nTrain
        f.open ( buf1,ios::in );
        f.read ( ( char* ) &nTrain,sizeof ( int ) );
        f.close();

        // nTest
        f.open ( buf2,ios::in );
        f.read ( ( char* ) &nTest,sizeof ( int ) );
        f.close();

        // targets
        float* tmp = new float[nTrain+nTest];
        f.open ( buf3,ios::in );
        f.read ( ( char* ) tmp,sizeof ( float ) *nTrain );
        f.close();
        trainTarget = new REAL[nTrain];
        for ( int i=0;i<nTrain;i++ )
            trainTarget[i] = tmp[i];
        testTarget = new REAL[nTest];
        //for(int i=0;i<nTest;i++)
        //    testTarget[i] = 3.7;
        f.open ( buf4,ios::in );
        f.read ( ( char* ) tmp,sizeof ( float ) *nTest );
        f.close();
        for ( int i=0;i<nTest;i++ )
            testTarget[i] = tmp[i];

        // get all the data files
        vector<string> files = Data::getDirectoryFileList ( buf0 );
        vector<string> predictionFiles;

        // read the *.dat files (prediction of probe+qual files)
        nFeat = 0;
        for ( int i=0;i<files.size();i++ )
        {
            string fileEnding = files[i].substr ( files[i].length()-4,4 );
            if ( fileEnding == ".dat" )
            {
                predictionFiles.push_back ( files[i] );
                nFeat++;
            }
        }

        cout<<"nFeat: "<<nFeat<<endl;
        cout<<"nClass: "<<nClass<<endl;
        cout<<"nTrain: "<<nTrain<<endl;
        cout<<"nTest: "<<nTest<<endl;

        // input features
        if ( Framework::getFrameworkMode() == 0 )
        {
            cout<<"allocate trainset: "<<nTrain * nFeat<<" elements"<<endl;
            train = new REAL[nTrain * nFeat];
            trainLabel = 0;
        }
        else
        {
            cout<<"allocate testset : "<< ( uint ) nTest * nFeat<<" elements"<<endl;
            test = new REAL[nTest * nFeat];
            testLabel = 0;
        }

        // predictions
        for ( int i=0;i<predictionFiles.size();i++ )
        {
            cout<<i<<"/"<< ( int ) predictionFiles.size() <<" ";
            f.open ( predictionFiles[i].c_str(), ios::in );
            f.read ( ( char* ) tmp, sizeof ( float ) * ( nTrain+nTest ) );
            f.close();
            double mean = 0.0;
            for ( int j=0;j<nTrain+nTest;j++ )
                mean += tmp[j];
            mean /= ( double ) ( nTrain+nTest );
            if ( mean > 1.0 && mean < 5.0 )
                cout<<"[clip] ";
            cout<<"mu:"<<mean<<" ";
            if ( Framework::getFrameworkMode() == 0 )
            {
                // train
                for ( int j=0;j<nTrain;j++ )
                {
                    train[j*nFeat + i] = tmp[j];
                    if ( mean > 1.0 && mean < 5.0 )
                    {
                        if ( train[j*nFeat + i] > 5.0 )
                            train[j*nFeat + i] = 5.0;
                        if ( train[j*nFeat + i] < 1.0 )
                            train[j*nFeat + i] = 1.0;
                    }
                }
            }
            else
            {
                // test
                for ( int j=0;j<nTest;j++ )
                {
                    test[j*nFeat + i] = tmp[j+nTrain];
                    if ( mean > 1.0 && mean < 5.0 )
                    {
                        if ( test[j*nFeat + i] > 5.0 )
                            test[j*nFeat + i] = 5.0;
                        if ( test[j*nFeat + i] < 1.0 )
                            test[j*nFeat + i] = 1.0;
                    }
                }
            }
            cout<<"Prediction file: "<<predictionFiles[i]<<endl;
        }

        if ( Framework::getFrameworkMode() == 0 )
            nTest = 0;
        else
            nTrain = 0;

        delete[] tmp;

    }
    
}

void DatasetReader::readPOKER	(	string	path,
		REAL *&	train,
		REAL *&	trainTarget,
		int *&	trainLabel,
		REAL *&	test,
		REAL *&	testTarget,
		int *&	testLabel,
		uint &	nTrain,
		uint &	nTest,
		int &	nClass,
		int &	nDomain,
		int &	nFeat,
		REAL	positiveTarget = 1.0,
		REAL	negativeTarget = -1.0
	)

Reads the POKER dataset (UCI)

24538333Bytes poker-hand-testing.data 613694Bytes poker-hand-training-true.data 5946Bytes poker-hand.names

Definition at line 4235 of file DatasetReader.cpp.

{
    cout<<"Read POKER from: "<<path<<endl;
    nDomain = 1;

    // define data type and files
    int targetColumn = 11;
    char columnType[] = "ddddddddddd";
    char enabledCol[] = "11111111111";
    const char* dataFiles[] = { ( new string ( path+"/poker-hand-training-true.data" ) )->c_str(), ( new string ( path+"/poker-hand-testing.data" ) )->c_str(),0};

    // === TRAIN SET ===
    getDataBounds ( dataFiles, ",", nFeat, nClass, nTrain, columnType, enabledCol, targetColumn, 0 );
    train = new REAL[nFeat*nTrain];
    trainLabel = new int[nTrain];
    getDataBounds ( dataFiles, ",", nFeat, nClass, nTrain, columnType, enabledCol, targetColumn, 0, true, train, trainLabel );

    // === TEST SET ===
    getDataBounds ( dataFiles, ",", nFeat, nClass, nTest, columnType, enabledCol, targetColumn, 1 );
    test = new REAL[nFeat*nTest];
    testLabel = new int[nTest];
    getDataBounds ( dataFiles, ",", nFeat, nClass, nTest, columnType, enabledCol, targetColumn, 1, true, test, testLabel );

    // make numerical test targets
    makeNumericTrainAndTestTargets ( nClass, nTrain, nTest, positiveTarget, negativeTarget, trainLabel, testLabel, trainTarget, testTarget );

}

void DatasetReader::readPRUDSYS	(	string	path,
		REAL *&	train,
		REAL *&	trainTarget,
		int *&	trainLabel,
		REAL *&	test,
		REAL *&	testTarget,
		int *&	testLabel,
		uint &	nTrain,
		uint &	nTest,
		int &	nClass,
		int &	nDomain,
		int &	nFeat,
		REAL	positiveTarget = 1.0,
		REAL	negativeTarget = -1.0
	)

PRUDSYS_DMC 2009: data mining cup Prudsys AG Trainset: dmc2009_train.txt (9239726 Bytes) Testset: dmc2009_forecast.txt (9308436 Bytes)

Definition at line 2279 of file DatasetReader.cpp.

{
    REAL* feat, *target;
    int* label, N;

    fstream f;
    if ( Framework::getFrameworkMode() == 1 )
    {
        f.open ( ( path+"/dmc2009_forecast.txt" ).c_str(), ios::in );
        nFeat = 1857+1;
        N = 2418;
        nClass = 1;
        nDomain = 8;
    }
    else
    {
        f.open ( ( path+"/dmc2009_train.txt" ).c_str(), ios::in );
        nFeat = 1857+1;
        N = 2394;
        nClass = 1;
        nDomain = 8;
    }

    feat = new REAL[N*nFeat];
    target = new REAL[N*nClass*nDomain];
    label = 0;

    // features and labels
    char *buf = new char[100000];
    f.getline ( buf,100000 );
    positiveTarget = -1e10;
    negativeTarget = 1e10;
    for ( int i=0;i<N;i++ )
    {
        f.getline ( buf,100000 );
        stringstream ss ( buf );
        REAL r;
        int cnt = 0;
        feat[nFeat*i + cnt] = 1.0;
        cnt++;
        while ( ss>>r )
        {
            if ( cnt < nFeat )
                feat[nFeat*i + cnt] = r;
            else if ( Framework::getFrameworkMode() == 0 )
                target[nDomain*nClass*i + cnt - nFeat] = r;
            else if ( Framework::getFrameworkMode() == 1 )
                target[nDomain*nClass*i + cnt - nFeat] = 0.0;
            cnt++;
        }
        if ( cnt != nFeat+nClass*nDomain && Framework::getFrameworkMode() == 0 )
            assert ( false );
    }
    f.close();
    delete[] buf;

    if ( Framework::getFrameworkMode() == 1 )
    {
        nTest = N;
        test = feat;
        testTarget = target;
        testLabel = label;
        train = 0;
        trainTarget = 0;
        trainLabel = 0;
        nTrain = 0;
    }
    else
    {
        nTrain = N;
        train = feat;
        trainTarget = target;
        trainLabel = label;
        test = 0;
        testTarget = 0;
        testLabel = 0;
        nTest = 0;
    }
}

void DatasetReader::readSATIMAGE	(	string	path,
		REAL *&	train,
		REAL *&	trainTarget,
		int *&	trainLabel,
		REAL *&	test,
		REAL *&	testTarget,
		int *&	testLabel,
		uint &	nTrain,
		uint &	nTest,
		int &	nClass,
		int &	nDomain,
		int &	nFeat,
		REAL	positiveTarget = 1.0,
		REAL	negativeTarget = -1.0
	)

Reads the SATIMAGE dataset (UCI)

5254Bytes sat.doc 525830Bytes sat.trn 236745Bytes sat.tst

Definition at line 4018 of file DatasetReader.cpp.

{
    cout<<"Read SATIMAGE from: "<<path<<endl;
    nDomain = 1;

    // define data type and files
    int targetColumn = 37;
    uint nTrainTmp;
    char columnType[] = "nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnd";
    char enabledCol[] = "1111111111111111111111111111111111111";
    const char* dataFiles[] = { ( new string ( path+"/sat.trn" ) )->c_str(), ( new string ( path+"/sat.tst" ) )->c_str(),0};

    // === TRAIN SET ===
    getDataBounds ( dataFiles, " ", nFeat, nClass, nTrain, columnType, enabledCol, targetColumn, 0 );
    train = new REAL[nFeat*nTrain];
    trainLabel = new int[nTrain];
    getDataBounds ( dataFiles, " ", nFeat, nClass, nTrain, columnType, enabledCol, targetColumn, 0, true, train, trainLabel );

    // === TEST SET ===
    getDataBounds ( dataFiles, " ", nFeat, nClass, nTest, columnType, enabledCol, targetColumn, 1 );
    test = new REAL[nFeat*nTest];
    testLabel = new int[nTest];
    getDataBounds ( dataFiles, " ", nFeat, nClass, nTest, columnType, enabledCol, targetColumn, 1, true, test, testLabel );

    // make numerical test targets
    makeNumericTrainAndTestTargets ( nClass, nTrain, nTest, positiveTarget, negativeTarget, trainLabel, testLabel, trainTarget, testTarget );

}

void DatasetReader::readSEGMENTATION	(	string	path,
		REAL *&	train,
		REAL *&	trainTarget,
		int *&	trainLabel,
		REAL *&	test,
		REAL *&	testTarget,
		int *&	testLabel,
		uint &	nTrain,
		uint &	nTest,
		int &	nClass,
		int &	nDomain,
		int &	nFeat,
		REAL	positiveTarget = 1.0,
		REAL	negativeTarget = -1.0
	)

Reads the SEGMENTATION dataset (UCI)

34481Bytes segmentation.data 2458Bytes segmentation.names 344723Bytes segmentation.test

Definition at line 4054 of file DatasetReader.cpp.

{
    cout<<"Read SEGMENTATION from: "<<path<<endl;
    nDomain = 1;

    // define data type and files
    int targetColumn = 1;
    uint nTrainTmp;
    char columnType[] = "dnnnnnnnnnnnnnnnnnnn";
    char enabledCol[] = "11111111111111111111";
    const char* dataFiles[] = { ( new string ( path+"/segmentation.data" ) )->c_str(), ( new string ( path+"/segmentation.test" ) )->c_str(),0};

    // === TRAIN SET ===
    getDataBounds ( dataFiles, ",", nFeat, nClass, nTrain, columnType, enabledCol, targetColumn, 0 );
    train = new REAL[nFeat*nTrain];
    trainLabel = new int[nTrain];
    getDataBounds ( dataFiles, ",", nFeat, nClass, nTrain, columnType, enabledCol, targetColumn, 0, true, train, trainLabel );

    // === TEST SET ===
    getDataBounds ( dataFiles, ",", nFeat, nClass, nTest, columnType, enabledCol, targetColumn, 1 );
    test = new REAL[nFeat*nTest];
    testLabel = new int[nTest];
    getDataBounds ( dataFiles, ",", nFeat, nClass, nTest, columnType, enabledCol, targetColumn, 1, true, test, testLabel );

    // make numerical test targets
    makeNumericTrainAndTestTargets ( nClass, nTrain, nTest, positiveTarget, negativeTarget, trainLabel, testLabel, trainTarget, testTarget );

}

void DatasetReader::readSONAR	(	string	path,
		REAL *&	train,
		REAL *&	trainTarget,
		int *&	trainLabel,
		REAL *&	test,
		REAL *&	testTarget,
		int *&	testLabel,
		uint &	nTrain,
		uint &	nTest,
		int &	nClass,
		int &	nDomain,
		int &	nFeat,
		REAL	positiveTarget = 1.0,
		REAL	negativeTarget = -1.0
	)

Reads the SONAR dataset (UCI)

87776Bytes sonar.all-data 5872Bytes sonar.names

Definition at line 4089 of file DatasetReader.cpp.

{
    cout<<"Read SONAR from: "<<path<<endl;
    nDomain = 1;

    // define data type and files
    int targetColumn = 61;
    uint nTrainTmp;
    char columnType[] = "nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnd";
    char enabledCol[] = "1111111111111111111111111111111111111111111111111111111111111";
    const char* dataFiles[] = { ( new string ( path+"/sonar.all-data" ) )->c_str(),0};

    // === TRAIN SET ===
    getDataBounds ( dataFiles, ",", nFeat, nClass, nTrainTmp, columnType, enabledCol, targetColumn, 0 );

    // allocate tmp mem
    REAL* trainTmp = new REAL[nTrainTmp * nFeat];
    int* trainLabelTmp = new int[nTrainTmp];

    // fill data
    getDataBounds ( dataFiles, ",", nFeat, nClass, nTrainTmp, columnType, enabledCol, targetColumn, 0, true, trainTmp, trainLabelTmp );

    // split train and testset from trainTmp
    splitRandomTestset ( 0.2, trainTmp, trainLabelTmp, nTrainTmp, nFeat, nClass, train, trainLabel, trainTarget, test, testLabel, testTarget, nTrain, nTest, positiveTarget, negativeTarget );

    delete[] trainTmp;
    delete[] trainLabelTmp;

}

void DatasetReader::readSPIDER	(	string	path,
		REAL *&	train,
		REAL *&	trainTarget,
		int *&	trainLabel,
		REAL *&	test,
		REAL *&	testTarget,
		int *&	testLabel,
		uint &	nTrain,
		uint &	nTest,
		int &	nClass,
		int &	nDomain,
		int &	nFeat,
		REAL	positiveTarget = 1.0,
		REAL	negativeTarget = -1.0
	)

Reads the artificial dataset generated by spider matlab framework

10CATBytes monks-1.test 2947Bytes monks-1.train

Definition at line 4341 of file DatasetReader.cpp.

{
    cout<<"Read SPIDER from: "<<path<<endl;
    nDomain = 1;
    nFeat = 3;
    nClass = 2;

    // the faster version for read-in
    int bufLen = 1024 * 1024;
    char *buf = new char[bufLen];

    // trainset
    nTrain = 0;
    fstream f ( ( path+"/train.data" ).c_str(), ios::in );
    while ( f.getline ( buf,bufLen ) )
        nTrain++;
    f.close();
    train = new REAL[3*nTrain];
    trainTarget = new REAL[2*nTrain];
    trainLabel = new int[nTrain];

    f.open ( ( path+"/train.data" ).c_str(), ios::in );
    nTrain = 0;
    while ( f.getline ( buf,bufLen ) )
    {
        sscanf ( buf,"%f %f %d",&train[3*nTrain],&train[3*nTrain+1],&trainLabel[nTrain] );
        train[3*nTrain+2] = 1.0;
        if ( trainLabel[nTrain] > 0 )
        {
            trainTarget[2*nTrain] = positiveTarget;
            trainTarget[2*nTrain+1] = negativeTarget;
            trainLabel[nTrain] = 0;
        }
        else
        {
            trainTarget[2*nTrain] = negativeTarget;
            trainTarget[2*nTrain+1] = positiveTarget;
            trainLabel[nTrain] = 1;
        }
        nTrain++;
    }
    f.close();

    // testset
    nTest = 0;
    f.open ( ( path+"/test.data" ).c_str(), ios::in );
    while ( f.getline ( buf,bufLen ) )
        nTest++;
    f.close();
    test = new REAL[3*nTest];
    testTarget = new REAL[2*nTest];
    testLabel = new int[nTest];

    f.open ( ( path+"/test.data" ).c_str(), ios::in );
    nTest = 0;
    while ( f.getline ( buf,bufLen ) )
    {
        sscanf ( buf,"%f %f %d",&test[3*nTest],&test[3*nTest+1],&testLabel[nTest] );
        test[3*nTest+2] = 1.0;
        if ( testLabel[nTrain] > 0 )
        {
            testTarget[2*nTest] = positiveTarget;
            testTarget[2*nTest+1] = negativeTarget;
            testLabel[nTest] = 0;
        }
        else
        {
            testTarget[2*nTest] = negativeTarget;
            testTarget[2*nTest+1] = positiveTarget;
            testLabel[nTest] = 1;
        }
        nTest++;
    }
    f.close();

    delete[] buf;

    /*
    // define data type and files
    int targetColumn = 3;
    char columnType[] = "nnd";
    char enabledCol[] = "111";
    const char* dataFiles[] = {(new string(path+"/train.data"))->c_str(),(new string(path+"/test.data"))->c_str(),0};

    bool addConstantOne = true;

    // === TRAIN SET ===
    getDataBounds(dataFiles, " ", nFeat, nClass, nTrain, columnType, enabledCol, targetColumn, 0, false, 0, 0, addConstantOne);
    train = new REAL[nFeat*nTrain];
    trainLabel = new int[nTrain];
    getDataBounds(dataFiles, " ", nFeat, nClass, nTrain, columnType, enabledCol, targetColumn, 0, true, train, trainLabel, addConstantOne);

    // === TEST SET ===
    getDataBounds(dataFiles, " ", nFeat, nClass, nTest, columnType, enabledCol, targetColumn, 1, false, 0, 0, addConstantOne);
    test = new REAL[nFeat*nTest];
    testLabel = new int[nTest];
    getDataBounds(dataFiles, " ", nFeat, nClass, nTest, columnType, enabledCol, targetColumn, 1, true, test, testLabel, addConstantOne);

    // make numerical test targets
    makeNumericTrainAndTestTargets(nClass, nTrain, nTest, positiveTarget, negativeTarget, trainLabel, testLabel, trainTarget, testTarget);
    */

    /*
    // simulate more domains
    nDomain = 3;
    int* trainLabelTmp = new int[nTrain*nDomain];
    for(int i=0;i<nTrain;i++)
        for(int d=0;d<nDomain;d++)
            trainLabelTmp[i*nDomain + d] = trainLabel[i];
    delete[] trainLabel;
    trainLabel = trainLabelTmp;

    int* testLabelTmp = new int[nTest*nDomain];
    for(int i=0;i<nTest;i++)
        for(int d=0;d<nDomain;d++)
            testLabelTmp[i*nDomain + d] = testLabel[i];
    delete[] testLabel;
    testLabel = testLabelTmp;

    // train targets
    trainTarget = new REAL[nClass*nDomain*nTrain];
    for(int i=0;i<nTrain;i++)
    {
        for(int d=0;d<nDomain;d++)
        {
            for(int j=0;j<nClass;j++)
                trainTarget[i*nClass*nDomain + d*nClass + j] = d==1?positiveTarget:negativeTarget;  // negative class labels
            trainTarget[i*nClass*nDomain + d*nClass + trainLabel[i*nDomain + d]] = d==1?negativeTarget:positiveTarget;  // positive class label
        }
    }

    // test targets
    testTarget = new REAL[nClass*nDomain*nTest];
    for(int i=0;i<nTest;i++)
    {
        for(int d=0;d<nDomain;d++)
        {
            for(int j=0;j<nClass;j++)
                testTarget[i*nClass*nDomain + d*nClass + j] = d==1?positiveTarget:negativeTarget;  // negative class labels
            testTarget[i*nClass*nDomain + d*nClass + testLabel[i*nDomain + d]] = d==1?negativeTarget:positiveTarget;  // positive class label
        }
    }
    */
}

void DatasetReader::readSURVIVAL	(	string	path,
		REAL *&	train,
		REAL *&	trainTarget,
		int *&	trainLabel,
		REAL *&	test,
		REAL *&	testTarget,
		int *&	testLabel,
		uint &	nTrain,
		uint &	nTest,
		int &	nClass,
		int &	nDomain,
		int &	nFeat,
		REAL	positiveTarget = 1.0,
		REAL	negativeTarget = -1.0
	)

Reads the SURVIVAL dataset (UCI)

3103Bytes haberman.data 1368Bytes haberman.names

Definition at line 4305 of file DatasetReader.cpp.

{
    cout<<"Read SURVIVAL from: "<<path<<endl;
    nDomain = 1;

    // define data type and files
    int targetColumn = 4;
    uint nTrainTmp;
    char columnType[] = "nnnd";
    char enabledCol[] = "1111";
    const char* dataFiles[] = { ( new string ( path+"/haberman.data" ) )->c_str(),0};

    // === TRAIN SET ===
    getDataBounds ( dataFiles, ",", nFeat, nClass, nTrainTmp, columnType, enabledCol, targetColumn, 0 );

    // allocate tmp mem
    REAL* trainTmp = new REAL[nTrainTmp * nFeat];
    int* trainLabelTmp = new int[nTrainTmp];

    // fill data
    getDataBounds ( dataFiles, ",", nFeat, nClass, nTrainTmp, columnType, enabledCol, targetColumn, 0, true, trainTmp, trainLabelTmp );

    // split train and testset from trainTmp
    splitRandomTestset ( 0.2, trainTmp, trainLabelTmp, nTrainTmp, nFeat, nClass, train, trainLabel, trainTarget, test, testLabel, testTarget, nTrain, nTest, positiveTarget, negativeTarget );

    delete[] trainTmp;
    delete[] trainLabelTmp;

}

void DatasetReader::readVEHICLE	(	string	path,
		REAL *&	train,
		REAL *&	trainTarget,
		int *&	trainLabel,
		REAL *&	test,
		REAL *&	testTarget,
		int *&	testLabel,
		uint &	nTrain,
		uint &	nTest,
		int &	nClass,
		int &	nDomain,
		int &	nFeat,
		REAL	positiveTarget = 1.0,
		REAL	negativeTarget = -1.0
	)

Reads the VEHICLE dataset (UCI)

55517Bytes train.data 6386Bytes vehicle.doc

Definition at line 4126 of file DatasetReader.cpp.

{
    cout<<"Read VEHICLE from: "<<path<<endl;
    nDomain = 1;

    // define data type and files
    int targetColumn = 19;
    uint nTrainTmp;
    char columnType[] = "nnnnnnnnnnnnnnnnnnd";
    char enabledCol[] = "1111111111111111111";
    const char* dataFiles[] = { ( new string ( path+"/train.data" ) )->c_str(),0};

    // === TRAIN SET ===
    getDataBounds ( dataFiles, " ", nFeat, nClass, nTrainTmp, columnType, enabledCol, targetColumn, 0 );

    // allocate tmp mem
    REAL* trainTmp = new REAL[nTrainTmp * nFeat];
    int* trainLabelTmp = new int[nTrainTmp];

    // fill data
    getDataBounds ( dataFiles, " ", nFeat, nClass, nTrainTmp, columnType, enabledCol, targetColumn, 0, true, trainTmp, trainLabelTmp );

    // split train and testset from trainTmp
    splitRandomTestset ( 0.2, trainTmp, trainLabelTmp, nTrainTmp, nFeat, nClass, train, trainLabel, trainTarget, test, testLabel, testTarget, nTrain, nTest, positiveTarget, negativeTarget );

    delete[] trainTmp;
    delete[] trainLabelTmp;

}

void DatasetReader::readVOTES	(	string	path,
		REAL *&	train,
		REAL *&	trainTarget,
		int *&	trainLabel,
		REAL *&	test,
		REAL *&	testTarget,
		int *&	testLabel,
		uint &	nTrain,
		uint &	nTest,
		int &	nClass,
		int &	nDomain,
		int &	nFeat,
		REAL	positiveTarget = 1.0,
		REAL	negativeTarget = -1.0
	)

Reads the VOTES dataset (UCI)

18171Bytes house-votes-84.data 6868Bytes house-votes-84.names

Definition at line 4162 of file DatasetReader.cpp.

{
    cout<<"Read VOTES from: "<<path<<endl;
    nDomain = 1;

    // define data type and files
    int targetColumn = 1;
    uint nTrainTmp;
    char columnType[] = "ddddddddddddddddd";
    char enabledCol[] = "11111111111111111";
    const char* dataFiles[] = { ( new string ( path+"/house-votes-84.data" ) )->c_str(),0};

    // === TRAIN SET ===
    getDataBounds ( dataFiles, ",", nFeat, nClass, nTrainTmp, columnType, enabledCol, targetColumn, 0 );

    // allocate tmp mem
    REAL* trainTmp = new REAL[nTrainTmp * nFeat];
    int* trainLabelTmp = new int[nTrainTmp];

    // fill data
    getDataBounds ( dataFiles, ",", nFeat, nClass, nTrainTmp, columnType, enabledCol, targetColumn, 0, true, trainTmp, trainLabelTmp );

    // split train and testset from trainTmp
    splitRandomTestset ( 0.2, trainTmp, trainLabelTmp, nTrainTmp, nFeat, nClass, train, trainLabel, trainTarget, test, testLabel, testTarget, nTrain, nTest, positiveTarget, negativeTarget );

    delete[] trainTmp;
    delete[] trainLabelTmp;

}

void DatasetReader::readWINE	(	string	path,
		REAL *&	train,
		REAL *&	trainTarget,
		int *&	trainLabel,
		REAL *&	test,
		REAL *&	testTarget,
		int *&	testLabel,
		uint &	nTrain,
		uint &	nTest,
		int &	nClass,
		int &	nDomain,
		int &	nFeat,
		REAL	positiveTarget = 1.0,
		REAL	negativeTarget = -1.0
	)

Reads the WINE dataset (UCI)

10782Bytes wine.data 3036Bytes wine.names

Definition at line 4198 of file DatasetReader.cpp.

{
    cout<<"Read WINE from: "<<path<<endl;
    nDomain = 1;

    // define data type and files
    int targetColumn = 1;
    uint nTrainTmp;
    char columnType[] = "dnnnnnnnnnnnnn";
    char enabledCol[] = "11111111111111";
    const char* dataFiles[] = { ( new string ( path+"/wine.data" ) )->c_str(),0};

    // === TRAIN SET ===
    getDataBounds ( dataFiles, ",", nFeat, nClass, nTrainTmp, columnType, enabledCol, targetColumn, 0 );

    // allocate tmp mem
    REAL* trainTmp = new REAL[nTrainTmp * nFeat];
    int* trainLabelTmp = new int[nTrainTmp];

    // fill data
    getDataBounds ( dataFiles, ",", nFeat, nClass, nTrainTmp, columnType, enabledCol, targetColumn, 0, true, trainTmp, trainLabelTmp );

    // split train and testset from trainTmp
    splitRandomTestset ( 0.2, trainTmp, trainLabelTmp, nTrainTmp, nFeat, nClass, train, trainLabel, trainTarget, test, testLabel, testTarget, nTrain, nTest, positiveTarget, negativeTarget );

    delete[] trainTmp;
    delete[] trainLabelTmp;

}

void DatasetReader::readYEAST	(	string	path,
		REAL *&	train,
		REAL *&	trainTarget,
		int *&	trainLabel,
		REAL *&	test,
		REAL *&	testTarget,
		int *&	testLabel,
		uint &	nTrain,
		uint &	nTest,
		int &	nClass,
		int &	nDomain,
		int &	nFeat,
		REAL	positiveTarget = 1.0,
		REAL	negativeTarget = -1.0
	)

Reads the YEAST dataset (UCI)

94976Bytes yeast.data 3313Bytes yeast.names

Definition at line 4269 of file DatasetReader.cpp.

{
    cout<<"Read YEAST from: "<<path<<endl;
    nDomain = 1;

    // define data type and files
    int targetColumn = 10;
    uint nTrainTmp;
    char columnType[] = "dnnnnnnnnd";
    char enabledCol[] = "0111111111";
    const char* dataFiles[] = { ( new string ( path+"/yeast.data" ) )->c_str(),0};

    // === TRAIN SET ===
    getDataBounds ( dataFiles, "  ", nFeat, nClass, nTrainTmp, columnType, enabledCol, targetColumn, 0 );

    // allocate tmp mem
    REAL* trainTmp = new REAL[nTrainTmp * nFeat];
    int* trainLabelTmp = new int[nTrainTmp];

    // fill data
    getDataBounds ( dataFiles, "  ", nFeat, nClass, nTrainTmp, columnType, enabledCol, targetColumn, 0, true, trainTmp, trainLabelTmp );

    // split train and testset from trainTmp
    splitRandomTestset ( 0.2, trainTmp, trainLabelTmp, nTrainTmp, nFeat, nClass, train, trainLabel, trainTarget, test, testLabel, testTarget, nTrain, nTest, positiveTarget, negativeTarget );

    delete[] trainTmp;
    delete[] trainLabelTmp;

}

void DatasetReader::splitRandomTestset	(	REAL	percentTest,
		REAL *	data,
		int *	labels,
		int	nData,
		int	nFeat,
		int	nClass,
		REAL *&	train,
		int *&	trainLabel,
		REAL *&	trainTarget,
		REAL *&	test,
		int *&	testLabel,
		REAL *&	testTarget,
		uint &	nTrain,
		uint &	nTest,
		REAL	positiveTarget,
		REAL	negativeTarget,
		bool	noRandom = false
	)

for split a random train and testset from data

Definition at line 4811 of file DatasetReader.cpp.

{
    // split the train and test set
    if ( noRandom )
        cout<<"take the last percentTest:"<<100.0*percentTest<<"[%]"<<endl;
    else
        cout<<"random percentTest:"<<100.0*percentTest<<"[%]"<<endl;

    // set train and test bounds
    nTrain = 0;
    nTest = 0;
    srand ( getRandomSeed() );
    for ( int i=0;i<nData;i++ )
    {
        REAL r = ( double ) rand() / ( double ) RAND_MAX;
        if ( noRandom ) // take the last x as testset
            r = ( double ) i/ ( double ) nData< ( 1.0 - percentTest ) ?1.0:0.0;
        if ( r < percentTest )
            nTest++;
        else
            nTrain++;
    }
    cout<<"nTrain:"<<nTrain<<endl;
    cout<<"nTest:"<<nTest<<endl;

    // allocate mem
    train = new REAL[nTrain * nFeat];
    trainLabel = new int[nTrain];
    test = new REAL[nTest * nFeat];
    testLabel = new int[nTest];

    // fill train and test set
    nTrain = 0;
    nTest = 0;
    srand ( getRandomSeed() );
    for ( int i=0;i<nData;i++ )
    {
        REAL r = ( double ) rand() / ( double ) RAND_MAX;
        if ( noRandom ) // take the last x as testset
            r = ( double ) i/ ( double ) nData< ( 1.0 - percentTest ) ?1.0:0.0;
        if ( r < percentTest )
        {
            for ( int j=0;j<nFeat;j++ )
                test[nTest*nFeat + j] = data[i*nFeat + j];
            testLabel[nTest] = labels[i];
            nTest++;
        }
        else
        {
            for ( int j=0;j<nFeat;j++ )
                train[nTrain*nFeat + j] = data[i*nFeat + j];
            trainLabel[nTrain] = labels[i];
            nTrain++;
        }
    }

    // check for NANs or INFs or too large numbers
    for ( int i=0;i<nTrain*nFeat;i++ )
        if ( isnan ( train[i] ) || isinf ( train[i] ) || train[i]>1e10 || train[i]<-1e10 )
        {
            cout<<"train["<<i<<"]:"<<train[i]<<endl;
            assert ( false );
        }

    for ( int i=0;i<nTest*nFeat;i++ )
        if ( isnan ( test[i] ) || isinf ( test[i] ) || test[i]>1e10 || test[i]<-1e10 )
        {
            cout<<"test["<<i<<"]:"<<test[i]<<endl;
            assert ( false );
        }

    makeNumericTrainAndTestTargets ( nClass, nTrain, nTest, positiveTarget, negativeTarget, trainLabel, testLabel, trainTarget, testTarget );
}

---

The documentation for this class was generated from the following files:

• ELF/DatasetReader.h
• ELF/DatasetReader.cpp