Autoencoder.cpp

#include "Autoencoder.h"

extern StreamOutput cout;

Autoencoder::Autoencoder()
{
    cout<<"Autoencoder"<<endl;
    // init member vars
    m_inputs = 0;
    m_nn = 0;
    m_epoch = 0;
    m_nrLayer = 0;
    m_batchSize = 0;
    m_offsetOutputs = 0;
    m_scaleOutputs = 0;
    m_initWeightFactor = 0;
    m_learnrate = 0;
    m_learnrateMinimum = 0;
    m_learnrateSubtractionValueAfterEverySample = 0;
    m_learnrateSubtractionValueAfterEveryEpoch = 0;
    m_momentum = 0;
    m_weightDecay = 0;
    m_minUpdateErrorBound = 0;
    m_etaPosRPROP = 0;
    m_etaNegRPROP = 0;
    m_minUpdateRPROP = 0;
    m_maxUpdateRPROP = 0;
    m_enableRPROP = 0;
    m_useBLASforTraining = 0;
    m_enableL1Regularization = 0;
    m_enableErrorFunctionMAE = 0;
    m_isFirstEpoch = 0;
    m_meanRBM = 0;
    m_stdRBM = 0;
    m_minTuninigEpochs = 0;
    m_maxTuninigEpochs = 0;
    m_nFixEpochs = -1;
}

Autoencoder::~Autoencoder()
{
    cout<<"descructor Autoencoder"<<endl;

    for ( int i=0;i<m_nCross+1;i++ )
    {
        if ( m_nn[i] )
            delete m_nn[i];
        m_nn[i] = 0;
    }
    delete[] m_nn;

    if ( m_isFirstEpoch )
        delete[] m_isFirstEpoch;
    m_isFirstEpoch = 0;
}

void Autoencoder::readMaps()
{
    cout<<"Read dsc maps (standard values)"<<endl;

}

double Autoencoder::train()
{
    cout<<"Start train Autoencoder"<<endl;

    // read standard and specific values
    readSpecificMaps();

    modelInit();

    if ( m_nFixEpochs == -1 )
    {
        cout<<endl<<"============================ START TRAIN (param tuning) ============================="<<endl<<endl;
        cout<<"Parameters to tune:"<<endl;

        addEpochParameter ( &m_epoch, "epoch" );

        // start the structured searcher
        cout<<"(min|max. epochs: "<<m_minTuninigEpochs<<"|"<<m_maxTuninigEpochs<<")"<<endl;
        expSearcher ( m_minTuninigEpochs, m_maxTuninigEpochs, 3, 1, 0.8, true, false );

        cout<<endl<<"============================ END auto-optimize ============================="<<endl<<endl;
    }

    cout<<"Update model on whole training set"<<endl<<endl;
    // retrain the model with whole trainingset (disable cross validation)
    if ( m_enableSaveMemory )
        fillNCrossValidationSet ( m_nCross );

    modelUpdate ( m_train[m_nCross], m_trainTarget[m_nCross], m_nTrain, m_nCross );
    saveWeights();

    if ( m_enableSaveMemory )
        freeNCrossValidationSet ( m_nCross );

    return 0.0;
}

void Autoencoder::readDataset ( Data* data, string datasetName )
{
    cout<<"Read data set and run the trained autoencoder"<<endl;

    data->readDataset ( datasetName );

    // load the normalizations
    cout<<"load the 0..1 normalizations"<<endl;
    REAL* mean = new REAL[data->m_nFeatures];
    REAL* std = new REAL[data->m_nFeatures];
    string meanName = data->m_datasetPath + "/" + data->m_tempPath + "/AutoencoderDataMean.dat";
    string stdName = data->m_datasetPath + "/" + data->m_tempPath + "/AutoencoderDataStd.dat";
    cout<<"meanName:"<<meanName<<endl<<"stdName:"<<stdName<<endl;
    fstream fMean ( meanName.c_str(),ios::in );
    fstream fStd ( stdName.c_str(),ios::in );
    if ( fMean.is_open() == false || fStd.is_open() == false )
        assert ( false );
    fMean.read ( ( char* ) mean, sizeof ( REAL ) *data->m_nFeatures );
    fStd.read ( ( char* ) std, sizeof ( REAL ) *data->m_nFeatures );
    fMean.close();
    fStd.close();

    // normalize train data
    cout<<"normalize train data"<<endl;
    for ( int i=0;i<data->m_nTrain;i++ )
        for ( int j=0;j<data->m_nFeatures;j++ )
        {
            data->m_trainOrig[j+i*data->m_nFeatures] = ( data->m_trainOrig[j+i*data->m_nFeatures] - mean[j] ) / std[j];
            REAL v = data->m_trainOrig[j+i*data->m_nFeatures];
            if ( v > 1.0 || v < 0.0 )
                cout<<"warning (0>v>1), transformed train value:"<<v<<endl;
        }

    // normalize test data
    cout<<"normalize test data"<<endl;
    for ( int i=0;i<data->m_nTest;i++ )
        for ( int j=0;j<data->m_nFeatures;j++ )
        {
            data->m_testOrig[j+i*data->m_nFeatures] = ( data->m_testOrig[j+i*data->m_nFeatures] - mean[j] ) / std[j];
            REAL v = data->m_testOrig[j+i*data->m_nFeatures];
            if ( v > 1.0 || v < 0.0 )
                cout<<"warning (0>v>1), transformed test value:"<<v<<endl;
        }

    delete[] mean;
    delete[] std;

    loadWeights();

    // new data
    cout<<"allocate new data matrices: #features:"<<m_nClass<<endl;
    REAL* trainOrig = new REAL[m_nClass * data->m_nTrain];
    REAL* testOrig = new REAL[m_nClass * data->m_nTest];

    // calculate dim. reduction
    time_t t0 = time ( 0 );
    cout<<"calculate dim. reduction: train set (size:"<<data->m_nTrain<<") "<<flush;
    predictAllOutputs ( data->m_trainOrig, trainOrig, data->m_nTrain, 0 );
    cout<<time ( 0 )-t0<<"[s]"<<endl;
    cout<<"calculate dim. reduction: test set (size:"<<data->m_nTest<<") "<<flush;
    t0 = time ( 0 );
    predictAllOutputs ( data->m_testOrig, testOrig, data->m_nTest, 0 );
    cout<<time ( 0 )-t0<<"[s]"<<endl;

    delete[] data->m_trainOrig;
    delete[] data->m_testOrig;

    data->m_trainOrig = trainOrig;
    data->m_testOrig = testOrig;
    data->m_nFeatures = m_nClass;

    // write new datasets
    // load the normalizations
    cout<<"write new datasets (AutoencoderDataTrain/Test/Targets)"<<endl;

    string trainName = data->m_datasetPath + "/" + data->m_tempPath + "/AutoencoderDataTrain.dat";
    fstream fTrain ( trainName.c_str(),ios::out );
    fTrain.write ( ( char* ) trainOrig, sizeof ( REAL ) *m_nClass * data->m_nTrain );
    fTrain.close();

    string testName = data->m_datasetPath + "/" + data->m_tempPath + "/AutoencoderDataTest.dat";
    fstream fTest ( testName.c_str(),ios::out );
    fTest.write ( ( char* ) testOrig, sizeof ( REAL ) *m_nClass * data->m_nTest );
    fTest.close();

    string trainTargetName = data->m_datasetPath + "/" + data->m_tempPath + "/AutoencoderDataTrainTarget.dat";
    fstream fTrainTarget ( trainTargetName.c_str(),ios::out );
    fTrainTarget.write ( ( char* ) data->m_trainTargetOrig, sizeof ( REAL ) *data->m_nClass * data->m_nTrain );
    fTrainTarget.close();

    string testTargetName = data->m_datasetPath + "/" + data->m_tempPath + "/AutoencoderDataTestTarget.dat";
    fstream fTestTarget ( testTargetName.c_str(),ios::out );
    fTestTarget.write ( ( char* ) data->m_testTargetOrig, sizeof ( REAL ) *data->m_nClass * data->m_nTest );
    fTestTarget.close();
}

void Autoencoder::loadNormalizations()
{
    m_meanRBM = new REAL[m_nFeatures];
    m_stdRBM = new REAL[m_nFeatures];

    cout<<"load the 0..1 normalizations (length "<<m_nFeatures<<")"<<endl;
    string meanName = m_datasetPath + "/" + m_tempPath + "/AutoencoderDataMean.dat";
    string stdName = m_datasetPath + "/" + m_tempPath + "/AutoencoderDataStd.dat";
    cout<<"meanName:"<<meanName<<endl<<"stdName:"<<stdName<<endl;
    fstream fMean ( meanName.c_str(),ios::in );
    fstream fStd ( stdName.c_str(),ios::in );
    if ( fMean.is_open() == false || fStd.is_open() == false )
        assert ( false );
    fMean.read ( ( char* ) m_meanRBM, sizeof ( REAL ) *m_nFeatures );
    fStd.read ( ( char* ) m_stdRBM, sizeof ( REAL ) *m_nFeatures );
    fMean.close();
    fStd.close();
}

double Autoencoder::calcRMSEonProbe()
{
    double rmse = 0.0;
    uint rmseCnt = 0;
    int nThreads = m_maxThreadsInCross;  // get #available threads

    for ( int i=0;i<m_nCross;i+=nThreads ) // all cross validation sets
    {
        // predict the probeset
        int* nSamples = new int[nThreads];
        double* rmses = new double[nThreads];
        uint* rmseCnts = new uint[nThreads];
        REAL** predictionProbe = new REAL*[nThreads];
        for ( int t=0;t<nThreads;t++ )
        {
            nSamples[t] = m_probeSize[i+t];
            rmses[t] = 0.0;
            rmseCnts[t] = 0;
            predictionProbe[t] = new REAL[nSamples[t]*m_nFeatures];
        }

        // parallel training of the cross-validation sets with OPENMP
#pragma omp parallel for
        for ( int t=0;t<nThreads;t++ )
        {
            cout<<"."<<flush;
            if ( m_enableSaveMemory )
                fillNCrossValidationSet ( i+t );
            modelUpdate ( m_train[i+t], m_trainTarget[i+t], m_trainSize[i+t], i+t );

            // predict all samples
            for ( int j=0;j<nSamples[t];j++ )
                m_nn[i+t]->m_nnAuto->predictSingleInput ( m_probe[i+t] + j*m_nFeatures, predictionProbe[t] + j*m_nFeatures );

            for ( int j=0;j<nSamples[t]*m_nFeatures;j++ ) // error over all probe samples
            {
                REAL err = predictionProbe[t][j] - m_probe[i+t][j];
                rmses[t] += err * err;
                rmseCnts[t]++;
            }
            //cout<<"[p:"<<sqrt(rmses[t]/(double)rmseCnts[t])<<"]"<<flush;

            if ( m_enableSaveMemory )
                freeNCrossValidationSet ( i+t );
        }

        // calc rmse sums
        for ( int t=0;t<nThreads;t++ )
        {
            rmse += rmses[t];
            rmseCnt += rmseCnts[t];
        }

        delete[] nSamples;
        delete[] rmses;
        delete[] rmseCnts;
        for ( int j=0;j<nThreads;j++ )
            delete[] predictionProbe[j];
    }

    return sqrt ( rmse/ ( double ) rmseCnt );
}

void Autoencoder::readSpecificMaps()
{
    cout<<"Read specific maps"<<endl;

    m_minTuninigEpochs = m_intMap["minTuninigEpochs"];
    m_maxTuninigEpochs = m_intMap["maxTuninigEpochs"];

    // read dsc vars
    m_nrLayer = m_intMap["nrLayer"];
    m_batchSize = m_intMap["batchSize"];
    m_offsetOutputs = m_doubleMap["offsetOutputs"];
    m_scaleOutputs = m_doubleMap["scaleOutputs"];
    m_initWeightFactor = m_doubleMap["initWeightFactor"];
    m_learnrate = m_doubleMap["learnrate"];
    m_learnrateMinimum = m_doubleMap["learnrateMinimum"];
    m_learnrateSubtractionValueAfterEverySample = m_doubleMap["learnrateSubtractionValueAfterEverySample"];
    m_learnrateSubtractionValueAfterEveryEpoch = m_doubleMap["learnrateSubtractionValueAfterEveryEpoch"];
    m_momentum = m_doubleMap["momentum"];
    m_weightDecay = m_doubleMap["weightDecay"];
    m_minUpdateErrorBound = m_doubleMap["minUpdateErrorBound"];
    m_etaPosRPROP = m_doubleMap["etaPosRPROP"];
    m_etaNegRPROP = m_doubleMap["etaNegRPROP"];
    m_minUpdateRPROP = m_doubleMap["minUpdateRPROP"];
    m_maxUpdateRPROP = m_doubleMap["maxUpdateRPROP"];
    m_enableL1Regularization = m_boolMap["enableL1Regularization"];
    m_enableErrorFunctionMAE = m_boolMap["enableErrorFunctionMAE"];
    m_enableRPROP = m_boolMap["enableRPROP"];
    m_useBLASforTraining = m_boolMap["useBLASforTraining"];
    m_neuronsPerLayer = m_stringMap["neuronsPerLayer"];
    m_nFixEpochs = m_intMap["nFixEpochs"];
}

void Autoencoder::modelInit()
{
    // set up NNs
    // nCross + 1 (for retraining)
    if ( m_nn == 0 )
    {
        m_nn = new NNRBM*[m_nCross+1];
        for ( int i=0;i<m_nCross+1;i++ )
            m_nn[i] = 0;
    }
    for ( int i=0;i<m_nCross+1;i++ )
    {
        cout<<"Create a Neural Network ("<<i+1<<"/"<<m_nCross+1<<")"<<endl;
        if ( m_nn[i] == 0 )
            m_nn[i] = new NNRBM();
        m_nn[i]->setNrTargets ( m_nClass*m_nDomain );
        m_nn[i]->setNrInputs ( m_nFeatures );
        m_nn[i]->setNrExamplesTrain ( 0 );
        m_nn[i]->setNrExamplesProbe ( 0 );
        m_nn[i]->setTrainInputs ( 0 );
        m_nn[i]->setTrainTargets ( 0 );
        m_nn[i]->setProbeInputs ( 0 );
        m_nn[i]->setProbeTargets ( 0 );
        m_nn[i]->setGlobalEpochs ( 0 );

        // learn parameters
        m_nn[i]->setInitWeightFactor ( m_initWeightFactor );
        m_nn[i]->setLearnrate ( m_learnrate );
        m_nn[i]->setLearnrateMinimum ( m_learnrateMinimum );
        m_nn[i]->setLearnrateSubtractionValueAfterEverySample ( m_learnrateSubtractionValueAfterEverySample );
        m_nn[i]->setLearnrateSubtractionValueAfterEveryEpoch ( m_learnrateSubtractionValueAfterEveryEpoch );
        m_nn[i]->setMomentum ( m_momentum );
        m_nn[i]->setWeightDecay ( m_weightDecay );
        m_nn[i]->setMinUpdateErrorBound ( m_minUpdateErrorBound );
        m_nn[i]->setBatchSize ( m_batchSize );
        m_nn[i]->setMaxEpochs ( m_maxTuninigEpochs );
        m_nn[i]->setL1Regularization ( m_enableL1Regularization );
        m_nn[i]->enableErrorFunctionMAE ( m_enableErrorFunctionMAE );

        m_nn[i]->setRBMLearnParams ( m_doubleMap["rbmLearnrateWeights"], m_doubleMap["rbmLearnrateBiasVis"], m_doubleMap["rbmLearnrateBiasHid"], m_doubleMap["rbmWeightDecay"], m_doubleMap["rbmMaxEpochs"] );

        // set net inner stucture
        int nrLayer = m_nrLayer;
        int* neuronsPerLayer = Data::splitStringToIntegerList ( m_neuronsPerLayer, ',' );
        m_nn[i]->enableRPROP ( m_enableRPROP );
        m_nn[i]->setNNStructure ( nrLayer, neuronsPerLayer, true );

        m_nn[i]->setScaleOffset ( m_scaleOutputs, m_offsetOutputs );
        m_nn[i]->setRPROPPosNeg ( m_etaPosRPROP, m_etaNegRPROP );
        m_nn[i]->setRPROPMinMaxUpdate ( m_minUpdateRPROP, m_maxUpdateRPROP );
        m_nn[i]->setNormalTrainStopping ( true );
        m_nn[i]->useBLASforTraining ( m_useBLASforTraining );
        m_nn[i]->initNNWeights ( m_randSeed );
        delete[] neuronsPerLayer;

        cout<<endl<<endl;
    }

    if ( m_isFirstEpoch == 0 )
        m_isFirstEpoch = new bool[m_nCross+1];
    for ( int i=0;i<m_nCross+1;i++ )
        m_isFirstEpoch[i] = false;
}

void Autoencoder::predictAllOutputs ( REAL* rawInputs, REAL* outputs, uint nSamples, uint crossRun )
{
    if ( m_meanRBM == 0 && m_stdRBM == 0 )
    {
        // predict all samples
        for ( int i=0;i<nSamples;i++ )
            m_nn[crossRun]->m_nnAuto->predictSingleInput ( rawInputs + i*m_nFeatures, outputs + i*m_nClass*m_nDomain );
    }
    else
    {
        REAL* in = new REAL[m_nFeatures];
        for ( int i=0;i<nSamples;i++ )
        {
            for ( int j=0;j<m_nFeatures;j++ )
                in[j] = ( rawInputs[i*m_nFeatures+j] - m_meanRBM[j] ) / m_stdRBM[j];
            m_nn[crossRun]->m_nnAuto->predictSingleInput ( in, outputs + i*m_nClass*m_nDomain );
        }
        delete[] in;
    }
}

void Autoencoder::modelUpdate ( REAL* input, REAL* target, uint nSamples, uint crossRun )
{
    if ( m_isFirstEpoch[crossRun] == true )
    {
        m_nn[crossRun]->m_nnAuto->setTrainInputs ( input );
        m_nn[crossRun]->m_nnAuto->setTrainTargets ( input );
        m_nn[crossRun]->m_nnAuto->setNrExamplesTrain ( nSamples );
    }
    else
    {
        m_nn[crossRun]->setTrainInputs ( input );
        m_nn[crossRun]->setTrainTargets ( target );
        m_nn[crossRun]->setNrExamplesTrain ( nSamples );
    }

    if ( crossRun < m_nCross )
    {
        if ( m_isFirstEpoch[crossRun] == false )
        {
            m_isFirstEpoch[crossRun] = true;

            // start the layerwise RBM pretraining
            m_nn[crossRun]->rbmPretraining ( input, target, nSamples, m_nDomain*m_nClass, -1, crossRun );
        }
        else
        {
            // one gradient descent step (one epoch)
            m_nn[crossRun]->m_nnAuto->trainOneEpoch();
            stringstream s;
            s<<"[t:"<<sqrt ( m_nn[crossRun]->m_nnAuto->m_sumSquaredError/ ( double ) m_nn[crossRun]->m_nnAuto->m_sumSquaredErrorSamples ) <<"] ";
            cout<<s.str() <<flush;
            if ( crossRun == m_nCross - 1 )
                m_nn[crossRun]->m_nnAuto->printLearnrate();
        }
    }
    else
    {
        cout<<endl<<"Tune: Training of full trainset "<<endl;

        if ( m_isFirstEpoch[crossRun] == false )
        {
            m_isFirstEpoch[crossRun] = true;

            // start the layerwise RBM pretraining
            m_nn[crossRun]->rbmPretraining ( input, target, nSamples, m_nDomain*m_nClass );
            //m_nn[crossRun]->printMiddleLayerToFile("tmp/pre.txt", input, target, nSamples, m_nDomain*m_nClass);
            //m_nn[crossRun]->m_nnAuto->printAutoencoderWeightsToJavascript("tmp/autoPre.txt");
        }

        // retraining with fix number of epochs
        m_nn[crossRun]->m_nnAuto->setNormalTrainStopping ( false );
        int maxEpochs;
        if ( m_nFixEpochs != -1 )
            maxEpochs = m_nFixEpochs;
        else
            maxEpochs = m_epochParamBest[0];
        if ( maxEpochs == 0 )
            maxEpochs = 1;  // train at least one epoch
        cout<<"Best #epochs (on cross validation): "<<maxEpochs<<endl;
        m_nn[crossRun]->m_nnAuto->setMaxEpochs ( maxEpochs );

        // train the net
        int epochs = m_nn[crossRun]->m_nnAuto->trainNN();
        //m_nn[crossRun]->printMiddleLayerToFile("tmp/fine.txt", input, target, nSamples, m_nDomain*m_nClass);
        //m_nn[crossRun]->m_nnAuto->printAutoencoderWeightsToJavascript("tmp/autoFine.txt");
        cout<<endl;
    }
}

void Autoencoder::saveWeights()
{
    string name = m_datasetPath + "/" + m_tempPath + "/AutoencoderWeights.dat";
    if ( m_inRetraining )
        cout<<"Save:"<<name<<endl;
    REAL* w = m_nn[m_nCross]->m_nnAuto->getWeightPtr();
    vector<int> v = m_nn[m_nCross]->m_nnAuto->getEncoder();

    fstream f ( name.c_str(), ios::out );

    // #layers
    int l = v.size() - 2;
    f.write ( ( char* ) &l, sizeof ( int ) );

    // neurons per layer
    for ( int i=0;i<v.size()-1;i++ )
        f.write ( ( char* ) &v[i], sizeof ( int ) );

    // net scale/offset
    f.write ( ( char* ) &m_scaleOutputs, sizeof ( double ) );
    f.write ( ( char* ) &m_offsetOutputs, sizeof ( double ) );
    f.write ( ( char* ) &m_useBLASforTraining, sizeof ( bool ) );

    // number of weights
    int n = v[v.size()-1];
    f.write ( ( char* ) &n, sizeof ( int ) );

    // weights
    f.write ( ( char* ) w, sizeof ( REAL ) *n );

    f.close();
}

void Autoencoder::loadWeights()
{
    // load weights
    string name = m_datasetPath + "/" + m_tempPath + "/AutoencoderWeights.dat";
    cout<<"Load:"<<name<<endl;
    fstream f ( name.c_str(), ios::in );
    if ( f.is_open() == false )
        assert ( false );

    // new net
    m_nn = new NNRBM*[m_nCross+1];
    for ( int i=0;i<m_nCross+1;i++ )
        m_nn[i] = 0;
    m_nn[0] = new NNRBM();
    m_nn[0]->m_nnAuto = new NNRBM();

    // #layers
    int l;
    f.read ( ( char* ) &l, sizeof ( int ) );
    int* neur = new int[l-1];

    // #features
    int in;
    f.read ( ( char* ) &in, sizeof ( int ) );
    m_nn[0]->m_nnAuto->setNrInputs ( in );
    m_nFeatures = in;

    // neurons per layer
    for ( int i=0;i<l-1;i++ )
    {
        int n;
        f.read ( ( char* ) &n, sizeof ( int ) );
        neur[i] = n;
    }
    int ntar;
    f.read ( ( char* ) &ntar, sizeof ( int ) );
    m_nn[0]->m_nnAuto->setNrTargets ( ntar );
    m_nClass = ntar;
    m_nDomain = 1;

    // net scale/offset
    f.read ( ( char* ) &m_scaleOutputs, sizeof ( double ) );
    f.read ( ( char* ) &m_offsetOutputs, sizeof ( double ) );
    f.read ( ( char* ) &m_useBLASforTraining, sizeof ( bool ) );

    m_nn[0]->m_nnAuto->setNNStructure ( l, neur, true );
    m_nn[0]->m_nnAuto->setScaleOffset ( m_scaleOutputs, m_offsetOutputs );
    m_nn[0]->m_nnAuto->useBLASforTraining ( m_useBLASforTraining );
    m_nn[0]->m_nnAuto->setInitWeightFactor ( 0.0 );
    m_nn[0]->m_nnAuto->initNNWeights ( 0 );

    // number of weights
    int nw = m_nn[0]->m_nnAuto->getNrWeights();
    int nwFile;
    f.read ( ( char* ) &nwFile, sizeof ( int ) );
    if ( nw != nwFile )
        assert ( false );

    // weights
    REAL* ptr = m_nn[0]->m_nnAuto->getWeightPtr();
    f.read ( ( char* ) ptr, sizeof ( REAL ) *nw );

    delete[] neur;
    f.close();
}

string Autoencoder::templateGenerator ( int id, string preEffect, int nameID, bool blendStop )
{
    stringstream s;
    s<<"ALGORITHM=Autoencoder"<<endl;
    s<<"ID="<<id<<endl;
    s<<"TRAIN_ON_FULLPREDICTOR="<<preEffect<<endl;
    s<<"DISABLE=0"<<endl;
    s<<endl;
    s<<"[int]"<<endl;
    s<<"nrLayer=4"<<endl;
    s<<"batchSize=1"<<endl;
    s<<"minTuninigEpochs=30"<<endl;
    s<<"maxTuninigEpochs=100"<<endl;
    s<<"nFixEpochs=-1"<<endl;
    s<<endl;
    s<<"[double]"<<endl;
    s<<"initMaxSwing=1.0"<<endl;
    s<<endl;
    s<<"offsetOutputs=0.0"<<endl;
    s<<"scaleOutputs=1.2"<<endl;
    s<<endl;
    s<<"etaPosRPROP=1.005"<<endl;
    s<<"etaNegRPROP=0.99"<<endl;
    s<<"minUpdateRPROP=1e-8"<<endl;
    s<<"maxUpdateRPROP=1e-2"<<endl;
    s<<endl;
    s<<"initWeightFactor=1.0"<<endl;
    s<<"learnrate=1e-3"<<endl;
    s<<"learnrateMinimum=1e-5"<<endl;
    s<<"learnrateSubtractionValueAfterEverySample=0.0"<<endl;
    s<<"learnrateSubtractionValueAfterEveryEpoch=0.0"<<endl;
    s<<"momentum=0.0"<<endl;
    s<<"weightDecay=0.0"<<endl;
    s<<"minUpdateErrorBound=1e-6"<<endl;
    s<<endl;
    s<<"[bool]"<<endl;
    s<<"enableErrorFunctionMAE=0"<<endl;
    s<<"enableL1Regularization=0"<<endl;
    s<<"enableClipping=1"<<endl;
    s<<"enableTuneSwing=0"<<endl;
    s<<"useBLASforTraining=1"<<endl;
    s<<"enableRPROP=0"<<endl;
    s<<endl;
    s<<"minimzeProbe="<< ( !blendStop ) <<endl;
    s<<"minimzeProbeClassificationError=0"<<endl;
    s<<"minimzeBlend="<<blendStop<<endl;
    s<<"minimzeBlendClassificationError=0"<<endl;
    s<<endl;
    s<<"[string]"<<endl;
    s<<"neuronsPerLayer=30,20,40,30,100,-1"<<endl;

    return s.str();
}