AlgorithmExploration.cpp

#include "AlgorithmExploration.h"

extern StreamOutput cout;

AlgorithmExploration::AlgorithmExploration()
{
    cout<<"AlgorithmExploration"<<endl;
}

AlgorithmExploration::~AlgorithmExploration()
{
    cout<<"descructor AlgorithmExploration"<<endl;
}

void AlgorithmExploration::start()
{
    time_t t0 = time ( 0 );
    cout.setOutputFile ( "out.txt" );

    // available algorithms for exploration
    vector<string> algos;
    algos.push_back ( "LM" );
    algos.push_back ( "NN" );
    algos.push_back ( "KNN" );
    //algos.push_back("PR");
    algos.push_back ( "KRR" );

    map<string,string> algoDscMap;
    algoDscMap["LM"] = "LinearModel_1.dsc";
    algoDscMap["NN"] = "NeuralNetwork_1.dsc";
    algoDscMap["KNN"] = "KNearestNeighbor_1.dsc";
    algoDscMap["PR"] = "PolynomialRegression_1.dsc";
    algoDscMap["KRR"] = "KernelRidgeRegression_1.dsc";

    map<string,string> algoPredMap;
    algoPredMap["LM"] = "LinearModel_1.dat";
    algoPredMap["NN"] = "NeuralNetwork_1.dat";
    algoPredMap["KNN"] = "KNearestNeighbor_1.dat";
    algoPredMap["PR"] = "PolynomialRegression_1.dat";
    algoPredMap["KRR"] = "KernelRidgeRegression_1.dat";

    // available datasets for exploration
    vector<string> datasets;
    /*datasets.push_back("CREDIT");
    datasets.push_back("BALANCE");
    datasets.push_back("BREAST");
    datasets.push_back("DIABETES");
    datasets.push_back("GERMAN");
    datasets.push_back("GLASS");*/
    datasets.push_back ( "HEPATITIS" );
    /*datasets.push_back("IONOSPHERE");
    datasets.push_back("IRIS");
    datasets.push_back("SONAR");
    datasets.push_back("SURVIVAL");
    datasets.push_back("VEHICLE");
    datasets.push_back("VOTES");
    datasets.push_back("WINE");*/

    //datasets.push_back("MUSHROOM"); // LARGE*/
    //datasets.push_back("LETTER"); // LARGE
    //datasets.push_back("SATIMAGE");  // LARGE
    //datasets.push_back("ADULT");  // LARGE

    // log file names
    string logResidualFilename = "logResidual.txt";
    string logResidualCascadeFilename = "logResidualCascade.txt";
    string logCascadeFilename = "logCascade.txt";
    string logStackingFilename = "logStacking.txt";
    /*string model = algos[0] + ".txt";
    string logResidualFilename = "";
    string logResidualCascadeFilename = "";
    string logCascadeFilename = model;
    string logStackingFilename = "";
    */
    /*
    // enable ensemble training methods
    bool enableResidual = true;
    bool enableResidualCascade = true;
    bool enableCascade = true;
    bool enableStacking = true;
    */
    // bagging
    bool enableResidual = false;
    bool enableResidualCascade = false;
    bool enableCascade = false;
    bool enableStacking = true;


    // number of test splits
    int nTestSplits = 100;

    // generate random permutations
    m_trainList.clear();
    for ( int depth=1;depth<algos.size() +1;depth++ )
    {
        vector<string> stack;
        randPerm ( stack, algos, depth );
    }
    cout<<endl<<"Residual/Cascade train list"<<endl;
    for ( int i=0;i<m_trainList.size();i++ )
    {
        for ( int j=0;j<m_trainList[i].size();j++ )
            cout<<m_trainList[i][j]<<" ";
        cout<<endl;
    }

    // generate train list for stacking (no permutations)
    cout<<endl<<"Stacking train list"<<endl;
    vector<vector<string> > stackingTrainListTmp;
    vector<vector<string> > stackingTrainList;
    for ( int i=0;i<m_trainList.size();i++ )
    {
        stackingTrainListTmp.push_back ( m_trainList[i] );
        sort ( stackingTrainListTmp[i].begin(), stackingTrainListTmp[i].end() );

        bool found = false;
        for ( int j=0;j<stackingTrainList.size();j++ )
        {
            if ( stackingTrainList[j] == stackingTrainListTmp[i] )
                found = true;
        }
        if ( found == false )
        {
            stackingTrainList.push_back ( m_trainList[i] );
            int s = stackingTrainList.size();
            sort ( stackingTrainList[s-1].begin(), stackingTrainList[s-1].end() );
        }
    }
    for ( int i=0;i<stackingTrainList.size();i++ )
    {
        for ( int j=0;j<stackingTrainList[i].size();j++ )
            cout<<stackingTrainList[i][j]<<" ";
        cout<<endl;
    }

    // hide console output
    cout.disableAllOutputs();

    // through all datasets
    for ( int datasetCnt=0;datasetCnt<datasets.size();datasetCnt++ )
    {
        string path = datasets[datasetCnt] + "/";
        printf ( "\nLog: %s  (nTestsets:%d)\n", ( path + logResidualFilename ).c_str(),nTestSplits );
        printf ( "Log: %s  (nTestsets:%d)\n", ( path + logResidualCascadeFilename ).c_str(),nTestSplits );
        printf ( "Log: %s  (nTestsets:%d)\n", ( path + logCascadeFilename ).c_str(),nTestSplits );
        printf ( "Log: %s  (nTestsets:%d)\n", ( path + logStackingFilename ).c_str(),nTestSplits );
        fstream fAnalyzeResidual ( ( path + logResidualFilename ).c_str(),ios::out );
        fstream fAnalyzeResidualCascade ( ( path + logResidualCascadeFilename ).c_str(),ios::out );
        fstream fAnalyzeCascade ( ( path + logCascadeFilename ).c_str(),ios::out );
        fstream fAnalyzeStacking ( ( path + logStackingFilename ).c_str(),ios::out );
        fstream f;

        time_t runTime = time ( 0 );

        // through all test splits
        for ( int testCnt=0;testCnt<nTestSplits;testCnt++ )
        {
            printf ( " %d ",testCnt );
            fflush ( stdout );

            uint randomSeedSplit = time ( 0 ) + testCnt;

            // ========================================= RESIDUAL TRAINING ==========================================
            // through all possible algorithm setups
            for ( int run=0;run<m_trainList.size() && enableResidual;run++ )
            {
                uint randomSeed = randomSeedSplit + run;

                printf ( "r" );
                fflush ( stdout );

                // write Master.dsc file
                string master;
                vector<string> algoDscList;
                for ( int i=0;i<m_trainList[run].size();i++ )
                    algoDscList.push_back ( algoDscMap[m_trainList[run][i]] );
                bool clas = true;
                bool cascade = false;
                master = Scheduler::masterDscTemplateGenerator ( datasets[datasetCnt], clas, algoDscList, randomSeed, "LinearRegression", cascade );
                f.open ( ( path+"Master.dsc" ).c_str(),ios::out );
                f<<master;
                f.close();

                // write algorithm *.dsc files
                string preEffect = "";
                for ( int i=0;i<m_trainList[run].size();i++ )
                {
                    string algoStr;
                    if ( m_trainList[run][i] == "LM" )
                        algoStr = LinearModel::templateGenerator ( i+1, preEffect, 1, true );
                    else if ( m_trainList[run][i] == "NN" )
                        algoStr = NeuralNetwork::templateGenerator ( i+1, preEffect, 1, true );
                    else if ( m_trainList[run][i] == "KNN" )
                        algoStr = KNearestNeighbor::templateGenerator ( i+1, preEffect, 1, true );
                    else if ( m_trainList[run][i] == "PR" )
                        algoStr = PolynomialRegression::templateGenerator ( i+1, preEffect, 1, true );
                    else if ( m_trainList[run][i] == "KRR" )
                        algoStr = KernelRidgeRegression::templateGenerator ( i+1, preEffect, 1, true );
                    else
                        assert ( false );
                    preEffect = algoPredMap[m_trainList[run][i]];

                    //cout<<"Write:"<<path+algoDscMap[m_trainList[run][i]]<<endl;
                    f.open ( ( path+algoDscMap[m_trainList[run][i]] ).c_str(),ios::out );
                    f<<algoStr;
                    f.close();
                }

                // train the ensemble
                Scheduler s;
                s.readMasterDscFile ( datasets[datasetCnt], "Master.dsc" );

                s.train();
                s.predict();

                // bagging
                //Framework::setAdditionalStartupParameter("10");
                //s.bagging();

                // boosting
                //Framework::setAdditionalStartupParameter("20");
                //s.boosting();

                // save error
                REAL rmse = s.getPredictionRMSE();
                REAL classErr = s.getClassificationError();
                fAnalyzeResidual<<rmse<<" "<<classErr<<" ";
            }
            fAnalyzeResidual<<endl;


            // ========================================= RESIDUAL+CASCADE TRAINING ==========================================
            // through all possible algorithm setups
            for ( int run=0;run<m_trainList.size() && enableResidualCascade;run++ )
            {
                uint randomSeed = randomSeedSplit + run;

                printf ( "m" );
                fflush ( stdout );

                // write Master.dsc file
                string master;
                vector<string> algoDscList;
                for ( int i=0;i<m_trainList[run].size();i++ )
                    algoDscList.push_back ( algoDscMap[m_trainList[run][i]] );
                bool clas = true;
                bool cascade = true;
                master = Scheduler::masterDscTemplateGenerator ( datasets[datasetCnt], clas, algoDscList, randomSeed, "LinearRegression", cascade );
                f.open ( ( path+"Master.dsc" ).c_str(),ios::out );
                f<<master;
                f.close();

                // write algorithm *.dsc files
                string preEffect = "";
                for ( int i=0;i<m_trainList[run].size();i++ )
                {
                    string algoStr;
                    if ( m_trainList[run][i] == "LM" )
                        algoStr = LinearModel::templateGenerator ( i+1, preEffect, 1, true );
                    else if ( m_trainList[run][i] == "NN" )
                        algoStr = NeuralNetwork::templateGenerator ( i+1, preEffect, 1, true );
                    else if ( m_trainList[run][i] == "KNN" )
                        algoStr = KNearestNeighbor::templateGenerator ( i+1, preEffect, 1, true );
                    else if ( m_trainList[run][i] == "PR" )
                        algoStr = PolynomialRegression::templateGenerator ( i+1, preEffect, 1, true );
                    else if ( m_trainList[run][i] == "KRR" )
                        algoStr = KernelRidgeRegression::templateGenerator ( i+1, preEffect, 1, true );
                    else
                        assert ( false );
                    preEffect = algoPredMap[m_trainList[run][i]];

                    //cout<<"Write:"<<path+algoDscMap[m_trainList[run][i]]<<endl;
                    f.open ( ( path+algoDscMap[m_trainList[run][i]] ).c_str(),ios::out );
                    f<<algoStr;
                    f.close();
                }

                // train the ensemble
                Scheduler s;
                s.readMasterDscFile ( datasets[datasetCnt], "Master.dsc" );
                s.train();
                s.predict();

                // save error
                REAL rmse = s.getPredictionRMSE();
                REAL classErr = s.getClassificationError();
                fAnalyzeResidualCascade<<rmse<<" "<<classErr<<" ";
            }
            fAnalyzeResidualCascade<<endl;


            // ========================================= CASCADE TRAINING ==========================================
            // through all possible algorithm setups
            for ( int run=0;run<m_trainList.size() && enableCascade;run++ )
            {
                uint randomSeed = randomSeedSplit + run;

                printf ( "c" );
                fflush ( stdout );

                // write Master.dsc file
                string master;
                vector<string> algoDscList;
                for ( int i=0;i<m_trainList[run].size();i++ )
                    algoDscList.push_back ( algoDscMap[m_trainList[run][i]] );
                bool clas = true;
                bool cascade = true;
                master = Scheduler::masterDscTemplateGenerator ( datasets[datasetCnt], clas, algoDscList, randomSeed, "TakeLast", cascade );
                f.open ( ( path+"Master.dsc" ).c_str(),ios::out );
                f<<master;
                f.close();

                // write algorithm *.dsc files
                string preEffect = "";
                for ( int i=0;i<m_trainList[run].size();i++ )
                {
                    string algoStr;
                    if ( m_trainList[run][i] == "LM" )
                        algoStr = LinearModel::templateGenerator ( i+1, preEffect, 1, false );
                    else if ( m_trainList[run][i] == "NN" )
                        algoStr = NeuralNetwork::templateGenerator ( i+1, preEffect, 1, false );
                    else if ( m_trainList[run][i] == "KNN" )
                        algoStr = KNearestNeighbor::templateGenerator ( i+1, preEffect, 1, false );
                    else if ( m_trainList[run][i] == "PR" )
                        algoStr = PolynomialRegression::templateGenerator ( i+1, preEffect, 1, false );
                    else if ( m_trainList[run][i] == "KRR" )
                        algoStr = KernelRidgeRegression::templateGenerator ( i+1, preEffect, 1, true );
                    else
                        assert ( false );
                    //preEffect = algoPredMap[m_trainList[run][i]];

                    //cout<<"Write:"<<path+algoDscMap[m_trainList[run][i]]<<endl;
                    f.open ( ( path+algoDscMap[m_trainList[run][i]] ).c_str(),ios::out );
                    f<<algoStr;
                    f.close();
                }

                // train the ensemble
                Scheduler s;
                s.readMasterDscFile ( datasets[datasetCnt], "Master.dsc" );
                s.train();
                s.predict();

                // save error
                REAL rmse = s.getPredictionRMSE();
                REAL classErr = s.getClassificationError();
                fAnalyzeCascade<<rmse<<" "<<classErr<<" ";
            }
            fAnalyzeCascade<<endl;


            // ========================================= STACKING TRAINING ==========================================
            // through all possible algorithm setups
            for ( int run=0;run<stackingTrainList.size() && enableStacking;run++ )
            {
                uint randomSeed = randomSeedSplit + run;

                printf ( "s" );
                fflush ( stdout );

                // write Master.dsc file
                string master;
                vector<string> algoDscList;
                for ( int i=0;i<stackingTrainList[run].size();i++ )
                    algoDscList.push_back ( algoDscMap[stackingTrainList[run][i]] );
                bool clas = true;
                bool cascade = false;
                master = Scheduler::masterDscTemplateGenerator ( datasets[datasetCnt], clas, algoDscList, randomSeed, "LinearRegressionNonNeg", cascade );
                f.open ( ( path+"Master.dsc" ).c_str(),ios::out );
                f<<master;
                f.close();

                // write algorithm *.dsc files
                string preEffect = "";
                for ( int i=0;i<stackingTrainList[run].size();i++ )
                {
                    string algoStr;
                    if ( stackingTrainList[run][i] == "LM" )
                        algoStr = LinearModel::templateGenerator ( i+1, preEffect, 1, false );
                    else if ( stackingTrainList[run][i] == "NN" )
                        algoStr = NeuralNetwork::templateGenerator ( i+1, preEffect, 1, false );
                    else if ( stackingTrainList[run][i] == "KNN" )
                        algoStr = KNearestNeighbor::templateGenerator ( i+1, preEffect, 1, false );
                    else if ( stackingTrainList[run][i] == "PR" )
                        algoStr = PolynomialRegression::templateGenerator ( i+1, preEffect, 1, false );
                    else if ( stackingTrainList[run][i] == "KRR" )
                        algoStr = KernelRidgeRegression::templateGenerator ( i+1, preEffect, 1, false );
                    else
                        assert ( false );
                    //preEffect = algoPredMap[stackingTrainList[run][i]];

                    //cout<<"Write:"<<path+algoDscMap[stackingTrainList[run][i]]<<endl;
                    f.open ( ( path+algoDscMap[stackingTrainList[run][i]] ).c_str(),ios::out );
                    f<<algoStr;
                    f.close();
                }

                // train the ensemble
                Scheduler s;
                s.readMasterDscFile ( datasets[datasetCnt], "Master.dsc" );

                s.train();
                s.predict();

                // bagging
                //Framework::setAdditionalStartupParameter("50");
                //s.bagging();

                // boosting
                //Framework::setAdditionalStartupParameter("20");
                //s.boosting();

                // save error
                REAL rmse = s.getPredictionRMSE();
                REAL classErr = s.getClassificationError();
                fAnalyzeStacking<<rmse<<" "<<classErr<<" ";
            }
            fAnalyzeStacking<<endl;

        }

        printf ( " run: %d[s]\n", ( int ) ( time ( 0 )-runTime ) );

        fAnalyzeResidual.close();
        fAnalyzeCascade.close();
        fAnalyzeStacking.close();
    }

    printf ( "Finished in: %d[s]\n", ( int ) ( time ( 0 )-t0 ) );
}

void AlgorithmExploration::randPerm ( vector<string> algorithmStack, vector<string> availableAlgorithms, int maxDepth )
{
    if ( maxDepth == 0 ) // max depth reached
    {
        m_trainList.push_back ( algorithmStack );
        return;
    }

    int size = availableAlgorithms.size();
    for ( int i=0;i<size;i++ )
    {
        vector<string> stack = algorithmStack;
        stack.push_back ( availableAlgorithms[i] );
        vector<string> algos;
        for ( int j=0;j<size;j++ )
            if ( i != j )
                algos.push_back ( availableAlgorithms[j] );
        randPerm ( stack, algos, maxDepth-1 );
    }
}