AutomaticParameterTuner.cpp

#include "AutomaticParameterTuner.h"

extern StreamOutput cout;

AutomaticParameterTuner::AutomaticParameterTuner()
{
    // first enable both optimizers
    m_optimizeProbeRMSE = true;
    m_optimizeBlendRMSE = true;

    m_expSearchMaxEpochsBlend = -1;

    m_enableDebug = true;

    m_epochParamBreak = false;
}

AutomaticParameterTuner::~AutomaticParameterTuner()
{
}

void AutomaticParameterTuner::addEpochParameter ( int *param, string name )
{
    m_epochParam.push_back ( param );
    m_epochName.push_back ( name );
}

void AutomaticParameterTuner::addIntegerParameter ( int *param, string name, int min, int max )
{
    m_intParam.push_back ( param );
    m_intMin.push_back ( min );
    m_intMax.push_back ( max );
    m_intName.push_back ( name );
}

void AutomaticParameterTuner::addDoubleParameter ( double *param, string name, double min, double max )
{
    m_doubleParam.push_back ( param );
    m_doubleMin.push_back ( min );
    m_doubleMax.push_back ( max );
    m_doubleName.push_back ( name );
}

void AutomaticParameterTuner::removeEpochParameter ( string name )
{
    int pos = -1;
    for ( int i=0;i<m_epochName.size();i++ )
        if ( m_epochName[i] == name )
            pos = i;
    if ( pos != -1 )
    {
        //cout<<"Remove epoch parameter:"<<name<<endl;
        m_epochName.erase ( m_epochName.begin() +pos );
        m_epochParam.erase ( m_epochParam.begin() +pos );
    }
    else
        cout<<"Warning: epoch parameter:"<<name<<" cannot be removed"<<endl;
}

void AutomaticParameterTuner::removeIntegerParameter ( string name )
{
    int pos = -1;
    for ( int i=0;i<m_intName.size();i++ )
        if ( m_intName[i] == name )
            pos = i;
    if ( pos != -1 )
    {
        //cout<<"Remove int parameter:"<<name<<endl;
        m_intName.erase ( m_intName.begin() +pos );
        m_intParam.erase ( m_intParam.begin() +pos );
        m_intMin.erase ( m_intMin.begin() +pos );
        m_intMax.erase ( m_intMax.begin() +pos );
    }
    else
        cout<<"Warning: int parameter:"<<name<<" cannot be removed"<<endl;
}

void AutomaticParameterTuner::removeDoubleParameter ( string name )
{
    int pos = -1;
    for ( int i=0;i<m_doubleName.size();i++ )
        if ( m_doubleName[i] == name )
            pos = i;
    if ( pos != -1 )
    {
        //cout<<"Remove double parameter:"<<name<<endl;
        m_doubleName.erase ( m_doubleName.begin() +pos );
        m_doubleParam.erase ( m_doubleParam.begin() +pos );
        m_doubleMin.erase ( m_doubleMin.begin() +pos );
        m_doubleMax.erase ( m_doubleMax.begin() +pos );
    }
    else
        cout<<"Warning: double parameter:"<<name<<" cannot be removed"<<endl;
}

void AutomaticParameterTuner::simpleStochasticParameterFinder ( double minProbeImpro, int maxProbeEpochsWithoutImpro, double minBlendImpro, int maxBlendEpochsWithoutImpro, double stdDev )
{
    vector<double> bestDoubleParam;
    vector<int> bestIntParam;

    bestDoubleParam.resize ( m_doubleParam.size() );
    for ( int i=0; i < m_doubleParam.size(); i++ )
        bestDoubleParam[i] = *m_doubleParam[i];
    bestIntParam.resize ( m_intParam.size() );
    for ( int i=0; i < m_intParam.size(); i++ )
        bestIntParam[i] = *m_intParam[i];


    int cnt=0;
    double bestRmse = 10.0;

    if ( m_optimizeProbeRMSE == true )
    {
        while ( cnt < maxProbeEpochsWithoutImpro )
        {
            for ( int i=0; i < m_doubleParam.size(); i++ )
                *m_doubleParam[i] = bestDoubleParam[i];
            for ( int i=0; i < m_intParam.size(); i++ )
                *m_intParam[i] = bestIntParam[i];

            long ticks = clock();
            int sel = rand() % ( m_doubleParam.size() + m_intParam.size() );

            if ( sel < m_doubleParam.size() )
            {
                *m_doubleParam[sel] = ( double ) NumericalTools::getNormRandomNumber ( bestDoubleParam[sel], max ( fabs ( bestDoubleParam[sel] ) * stdDev, 0.001 ) );
                *m_doubleParam[sel] = NumericalTools::clipValue ( *m_doubleParam[sel], m_doubleMin[sel], m_doubleMax[sel] );
            }
            else
            {
                sel -= m_doubleParam.size();

                *m_intParam[sel] = ( int ) NumericalTools::getNormRandomNumber ( ( double ) bestIntParam[sel], max ( fabs ( ( double ) bestIntParam[sel] ) * stdDev, 1.0 ) );

                if ( *m_intParam[sel] < m_intMin[sel] )
                    *m_intParam[sel] = m_intMin[sel];
                if ( *m_intParam[sel] > m_intMax[sel] )
                    *m_intParam[sel] = m_intMax[sel];
            }


            double rmse = calcRMSEonProbe();

            if ( rmse < bestRmse )
            {
                if ( bestRmse - rmse > minProbeImpro )
                {
                    cout<<"*";
                    cnt=0;
                }

                bestRmse = rmse;
                for ( int i=0; i < m_doubleParam.size(); i++ )
                    bestDoubleParam[i] = *m_doubleParam[i];
                for ( int i=0; i < m_intParam.size(); i++ )
                    bestIntParam[i] = *m_intParam[i];
                cout<<"* ";
            }

            ticks = clock() - ticks;

            cnt++;

            // output
            cout<< ( float ) ticks / ( float ) CLOCKS_PER_SEC<<" s | ERR: "<<rmse;
            for ( int i=0; i < m_doubleParam.size(); i++ )
                cout<<" | "<<m_doubleName[i]<<": "<<*m_doubleParam[i];
            for ( int i=0; i < m_intParam.size(); i++ )
                cout<<" | "<<m_intName[i]<<": "<<*m_intParam[i];
            cout<<endl;
        }

        for ( int i=0; i < m_doubleParam.size(); i++ )
            *m_doubleParam[i] = bestDoubleParam[i];
        for ( int i=0; i < m_intParam.size(); i++ )
            *m_intParam[i] = bestIntParam[i];

        cout<<"bestParameters on Probe: ERR: "<<bestRmse<<" ";
        for ( int i=0; i < m_doubleParam.size(); i++ )
            cout<<" | "<<m_doubleName[i]<<": "<<*m_doubleParam[i];
        for ( int i=0; i < m_intParam.size(); i++ )
            cout<<" | "<<m_intName[i]<<": "<<*m_intParam[i];
        cout<<endl;
    }



    // ============== find the best Parameters on the blend =================
    cnt=0;
    bestRmse = 10.0;
    if ( m_optimizeBlendRMSE == true )
    {
        while ( cnt < maxBlendEpochsWithoutImpro )
        {
            for ( int i=0; i < m_doubleParam.size(); i++ )
                *m_doubleParam[i] = bestDoubleParam[i];
            for ( int i=0; i < m_intParam.size(); i++ )
                *m_intParam[i] = bestIntParam[i];

            long ticks = clock();
            int sel = rand() % ( m_doubleParam.size() + m_intParam.size() );

            if ( sel < m_doubleParam.size() )
            {
                *m_doubleParam[sel] = ( double ) NumericalTools::getNormRandomNumber ( bestDoubleParam[sel], max ( fabs ( bestDoubleParam[sel] ) * stdDev, 0.001 ) );
                *m_doubleParam[sel] = NumericalTools::clipValue ( *m_doubleParam[sel], m_doubleMin[sel], m_doubleMax[sel] );
            }
            else
            {
                sel -= m_doubleParam.size();

                *m_intParam[sel] = ( int ) NumericalTools::getNormRandomNumber ( ( double ) bestIntParam[sel], max ( fabs ( ( double ) bestIntParam[sel] ) * stdDev, 1.0 ) );

                if ( *m_intParam[sel] < m_intMin[sel] )
                    *m_intParam[sel] = m_intMin[sel];
                if ( *m_intParam[sel] > m_intMax[sel] )
                    *m_intParam[sel] = m_intMax[sel];
            }


            double rmse = calcRMSEonBlend();

            if ( rmse < bestRmse )
            {
                if ( bestRmse - rmse > minBlendImpro )
                {
                    cout<<"*";
                    cnt=0;
                }

                bestRmse = rmse;
                for ( int i=0; i < m_doubleParam.size(); i++ )
                    bestDoubleParam[i] = *m_doubleParam[i];
                for ( int i=0; i < m_intParam.size(); i++ )
                    bestIntParam[i] = *m_intParam[i];
                cout<<"* ";
            }

            ticks = clock() - ticks;

            cnt++;

            // output
            cout<< ( float ) ticks / ( float ) CLOCKS_PER_SEC<<" s | ERR: "<<rmse;
            for ( int i=0; i < m_doubleParam.size(); i++ )
                cout<<" | "<<m_doubleName[i]<<": "<<*m_doubleParam[i];
            for ( int i=0; i < m_intParam.size(); i++ )
                cout<<" | "<<m_intName[i]<<": "<<*m_intParam[i];
            cout<<endl;
        }

        for ( int i=0; i < m_doubleParam.size(); i++ )
            *m_doubleParam[i] = bestDoubleParam[i];
        for ( int i=0; i < m_intParam.size(); i++ )
            *m_intParam[i] = bestIntParam[i];

        cout<<"bestParameters on Blend: ERR: "<<bestRmse<<" ";
        for ( int i=0; i < m_doubleParam.size(); i++ )
            cout<<" | "<<m_doubleName[i]<<": "<<*m_doubleParam[i];
        for ( int i=0; i < m_intParam.size(); i++ )
            cout<<" | "<<m_intName[i]<<": "<<*m_intParam[i];
        cout<<endl;
    }
}

void AutomaticParameterTuner::setOptimizeProbeRmse ( bool enable )
{
    m_optimizeProbeRMSE = enable;
}

void AutomaticParameterTuner::setOptimizeBlendRmse ( bool enable )
{
    m_optimizeBlendRMSE = enable;
}

void AutomaticParameterTuner::setDebug ( bool en )
{
    m_enableDebug = en;
    //cout<<"Debug: "<<m_enableDebug<<endl;
}

void AutomaticParameterTuner::expSearchParams ( int minEpochs, int maxEpochs, int paramEpochs, int accelerationEpochs, double expInit )
{
    m_expSearchMinEpochs = minEpochs;
    m_expSearchMaxEpochs = maxEpochs;
    m_expSearchParamEpochs = paramEpochs;
    m_expSearchParamAccelerationEpochs = accelerationEpochs;
    m_expSearchEpoch = 0;
    m_expSearchVariationCnt = 0;

    m_epochParamPos = 0;
    m_expSearchDoubleParamPos = 0;
    m_expSearchIntParamPos = 0;
    m_expSearchErrorBest = 1e10;
    m_expSearchAcceleration = 0.8;
    m_expSearchAccelerationEpoch = 0;

    m_epochParamBest.clear();
    m_expSearchExponent.clear();
    m_expSearchDoubleParamBest.clear();
    m_expSearchIntParamBest.clear();
    m_expSearchParamDirection.clear();

    m_expInit = expInit;

    for ( int i=0;i<m_epochParam.size();i++ )
        m_epochParamBest.push_back ( *m_epochParam[i] );
    for ( int i=0;i<m_doubleParam.size();i++ )
    {
        m_expSearchExponent.push_back ( expInit );
        m_expSearchParamDirection.push_back ( false );
        m_expSearchDoubleParamBest.push_back ( *m_doubleParam[i] );
    }
    for ( int i=0;i<m_intParam.size();i++ )
    {
        m_expSearchExponent.push_back ( expInit );
        m_expSearchParamDirection.push_back ( false );
        m_expSearchIntParamBest.push_back ( *m_intParam[i] );
    }
}

bool AutomaticParameterTuner::expSearchChangeParams()
{
    m_expSearchTime = time ( 0 );
    m_expSearchEpoch++;

    if ( m_enableDebug )
        cout<<"(epoch="<<m_expSearchEpoch-1<<") "<<flush;

    // break training if error rises on epoch tuninig and larger as minEpochs
    if ( m_epochParamBreak &&  m_expSearchEpoch >= m_expSearchMinEpochs )
    {
        if ( m_enableDebug )
            cout<<"epoch training error rises. (#minEpochs:"<<m_expSearchMinEpochs<<", )"<<endl;
        return false;
    }

    // in the first epoch do nothing
    if ( m_expSearchEpoch==1 )
    {
        for ( int i=0;i<m_epochParam.size();i++ )
            if ( m_enableDebug )
                cout<<m_epochName[i]<<"="<< ( *m_epochParam[i] ) <<" ";
        for ( int i=0;i<m_doubleParam.size();i++ )
            if ( m_enableDebug )
                cout<<m_doubleName[i]<<"="<< ( *m_doubleParam[i] ) <<" ";
        for ( int i=0;i<m_intParam.size();i++ )
            if ( m_enableDebug )
                cout<<m_intName[i]<<"="<< ( *m_intParam[i] ) <<" ";
        return true;
    }

    // change a epoch parameter
    if ( m_epochParamPos < m_epochParam.size() )
    {
        // do the change
        *m_epochParam[m_epochParamPos] = *m_epochParam[m_epochParamPos] + 1;
    }

    // change a double parameter
    if ( m_expSearchDoubleParamPos < m_doubleParam.size() && m_epochParamPos == m_epochParam.size() )
    {
        int pos = m_expSearchDoubleParamPos;
        double exp = m_expSearchExponent[pos];
        bool dir = m_expSearchParamDirection[pos];
        double best = m_expSearchDoubleParamBest[pos];

        // do the change
        *m_doubleParam[pos] = best * ( dir? ( 1.0/exp ) : exp );

        // check boundaries
        if ( *m_doubleParam[pos] > m_doubleMax[pos] )
            *m_doubleParam[pos] = m_doubleMax[pos];
        if ( *m_doubleParam[pos] < m_doubleMin[pos] )
            *m_doubleParam[pos] = m_doubleMin[pos];
    }

    // change a int parameter
    if ( m_expSearchIntParamPos < m_intParam.size() && m_expSearchDoubleParamPos == m_doubleParam.size() )
    {
        int pos = m_expSearchIntParamPos;
        double exp = m_expSearchExponent[pos + m_doubleParam.size() ];
        bool dir = m_expSearchParamDirection[pos + m_doubleParam.size() ];
        int best = m_expSearchIntParamBest[pos];

        // do the change
        *m_intParam[pos] = ( int ) ( round ( ( double ) ( best ) * ( dir? ( 1.0/exp ) : exp ) ) );

        // if no change, force the change
        if ( best == *m_intParam[pos] )
            *m_intParam[pos] = dir? best+1 : best-1;

        // check boundaries
        if ( *m_intParam[pos] > m_intMax[pos] )
            *m_intParam[pos] = m_intMax[pos];
        if ( *m_intParam[pos] < m_intMin[pos] )
            *m_intParam[pos] = m_intMin[pos];
    }

    for ( int i=0;i<m_epochParam.size();i++ )
        if ( m_enableDebug )
            cout<<m_epochName[i]<<"="<< ( *m_epochParam[i] ) <<" ";
    for ( int i=0;i<m_doubleParam.size();i++ )
        if ( m_enableDebug )
            cout<<m_doubleName[i]<<"="<< ( *m_doubleParam[i] ) <<" ";
    for ( int i=0;i<m_intParam.size();i++ )
        if ( m_enableDebug )
            cout<<m_intName[i]<<"="<< ( *m_intParam[i] ) <<" ";

    // break, if maxEpochs is reached
    if ( m_expSearchEpoch > m_expSearchMaxEpochs )
    {
        m_expSearchEpoch = 0;
        // apply best
        for ( int i=0;i<m_epochParam.size();i++ )
            *m_epochParam[i] = m_epochParamBest[i];
        for ( int i=0;i<m_doubleParam.size();i++ )
            *m_doubleParam[i] = m_expSearchDoubleParamBest[i];
        for ( int i=0;i<m_intParam.size();i++ )
            *m_intParam[i] = m_expSearchIntParamBest[i];
        if ( m_enableDebug )
            cout<<"max. epochs reached."<<endl;
        return false;
    }

    return true;
}

void AutomaticParameterTuner::expSearchCheckErr ( double error )
{
    if ( m_enableDebug )
        cout<<"ERR="<<error;

    if ( m_enableDebug )
        cout<<" "<<time ( 0 )-m_expSearchTime<<"[s]";

    // in the first epoch do nothing
    if ( m_expSearchEpoch==1 )
    {
        saveBestPrediction();
        for ( int i=0;i<m_epochParam.size();i++ )
            m_epochParamBest[i] = *m_epochParam[i];
        for ( int i=0;i<m_doubleParam.size();i++ )
            m_expSearchDoubleParamBest[i] = *m_doubleParam[i];
        for ( int i=0;i<m_intParam.size();i++ )
            m_expSearchIntParamBest[i] = *m_intParam[i];
        m_expSearchErrorBest = error;
        if ( m_enableDebug )
            cout<<endl;
        return;
    }

    // if the error is lower than the old
    if ( error<m_expSearchErrorBest )
    {
        m_expSearchErrorBest = error;
        //cout<<"bestUpdate:"<<m_expSearchErrorBest<<" ";

        m_epochParamBreak = 0;

        if ( m_enableDebug )
            cout<<" !min! ";
        saveBestPrediction();

        // current in a epoch parameter
        if ( m_epochParamPos < m_epochParam.size() )
        {
            m_epochParamBest[m_epochParamPos] = *m_epochParam[m_epochParamPos];
        }

        // current in a double parameter
        if ( m_expSearchDoubleParamPos < m_doubleParam.size() && m_epochParamPos == m_epochParam.size() )
        {
            int pos = m_expSearchDoubleParamPos;
            double exp = m_expSearchExponent[pos];
            bool dir = m_expSearchParamDirection[pos];

            // new best value
            m_expSearchDoubleParamBest[pos] = *m_doubleParam[pos];

            // if we go in the right direction, increase step size
            if ( m_expSearchAccelerationEpoch >= m_expSearchParamAccelerationEpochs )
            {
                if ( m_enableDebug )
                    cout<<" accelerate ";
                m_expSearchExponent[pos] = pow ( m_expSearchExponent[pos], 1.0/m_expSearchAcceleration );
            }

        }

        // current in a int parameter
        if ( m_expSearchIntParamPos < m_intParam.size() && m_expSearchDoubleParamPos == m_doubleParam.size() )
        {
            int pos = m_expSearchIntParamPos;
            double exp = m_expSearchExponent[pos + m_doubleParam.size() ];
            bool dir = m_expSearchParamDirection[pos + m_doubleParam.size() ];

            // new best value
            m_expSearchIntParamBest[pos] = *m_intParam[pos];

            // if we go in the right direction, increase step size
            if ( m_expSearchAccelerationEpoch >= m_expSearchParamAccelerationEpochs )
            {
                if ( m_enableDebug )
                    cout<<" accelerate ";
                m_expSearchExponent[pos] = pow ( m_expSearchExponent[pos], 1.0/m_expSearchAcceleration );
            }
        }

        m_expSearchAccelerationEpoch++;
    }
    else  // error gets higher, so change search direction and decrease step size
    {
        // current in a epoch parameter
        if ( m_epochParamPos < m_epochParam.size() )
        {
            // break training
            m_epochParamBreak = true;
        }

        // current in a double parameter
        if ( m_expSearchDoubleParamPos < m_doubleParam.size() && m_epochParamPos == m_epochParam.size() )
        {
            int pos = m_expSearchDoubleParamPos;
            bool dir = m_expSearchParamDirection[pos];
            m_expSearchParamDirection[pos] = !m_expSearchParamDirection[pos];
            m_expSearchExponent[pos] = pow ( m_expSearchExponent[pos], m_expSearchAcceleration );
        }

        // current in a int parameter
        if ( m_expSearchIntParamPos < m_intParam.size() && m_expSearchDoubleParamPos == m_doubleParam.size() )
        {
            int pos = m_expSearchIntParamPos;
            bool dir = m_expSearchParamDirection[pos + m_doubleParam.size() ];
            m_expSearchParamDirection[pos + m_doubleParam.size() ] = !m_expSearchParamDirection[pos + m_doubleParam.size() ];
            m_expSearchExponent[pos + m_doubleParam.size() ] = pow ( m_expSearchExponent[pos + m_doubleParam.size() ], m_expSearchAcceleration );
        }

        m_expSearchAccelerationEpoch = 0;
    }


    // goto next parameter, current in a epoch parameter
    if ( m_epochParamPos < m_epochParam.size() )
    {
        m_epochParamPos++;

        // if no integer and no double parameter available
        if ( m_doubleParam.size() == 0 && m_intParam.size() == 0 && m_epochParamPos == m_epochParam.size() )
            m_epochParamPos = 0;
    }

    // goto next parameter, current in a double parameter
    if ( m_expSearchDoubleParamPos < m_doubleParam.size() && m_epochParamPos == m_epochParam.size() )
    {
        // change to next parameter
        m_expSearchVariationCnt++;
        if ( m_expSearchVariationCnt==m_expSearchParamEpochs )
        {
            m_expSearchAccelerationEpoch = 0;
            m_expSearchVariationCnt = 0;
            *m_doubleParam[m_expSearchDoubleParamPos] = m_expSearchDoubleParamBest[m_expSearchDoubleParamPos];  // apply best
            m_expSearchDoubleParamPos++;
        }

        // if no integer parameter available
        if ( m_intParam.size() == 0 && m_expSearchDoubleParamPos == m_doubleParam.size() )
            m_expSearchDoubleParamPos = 0;
    }

    // goto next parameter, current in a int parameter
    if ( m_expSearchIntParamPos < m_intParam.size() && m_expSearchDoubleParamPos == m_doubleParam.size() )
    {
        // change to next parameter
        m_expSearchVariationCnt++;
        if ( m_expSearchVariationCnt==m_expSearchParamEpochs )
        {
            m_expSearchAccelerationEpoch = 0;
            m_expSearchVariationCnt = 0;
            *m_intParam[m_expSearchIntParamPos] = m_expSearchIntParamBest[m_expSearchIntParamPos];  // apply best
            m_expSearchIntParamPos++;
        }

        if ( m_expSearchIntParamPos == m_intParam.size() )
        {
            m_epochParamPos = 0;
            m_expSearchIntParamPos = 0;
            m_expSearchDoubleParamPos = 0;
        }
    }

    if ( m_enableDebug )
        cout<<endl;
}

void AutomaticParameterTuner::expSearcher ( int minEpochs, int maxEpochs, int paramEpochs, int accelerationEpochs, double expInit, bool enableProbe, bool enableBlend )
{
    if ( enableProbe==0 && enableBlend==0 )
    {
        cout<<"Warning: both enableProbe and enableBlend are 0"<<endl;
        cout<<"Skip tuning"<<endl;
    }

    if ( maxEpochs == 0 )
        return;

    m_epochParamBreak = false;

    expSearchParams ( minEpochs, maxEpochs, paramEpochs, accelerationEpochs, expInit );
    m_optimizeProbeRMSE = enableProbe;
    m_optimizeBlendRMSE = enableBlend;

    // minimize on probe
    if ( m_optimizeProbeRMSE )
    {
        double error;
        while ( expSearchChangeParams() )
        {
            error = calcRMSEonProbe();
            expSearchCheckErr ( error );
        }
        if ( m_enableDebug )
            cout<<"expSearchErrorBest:"<<m_expSearchErrorBest<<"  error:"<<error<<endl;
    }
    // minimize on blend
    if ( m_optimizeBlendRMSE )
    {
        // if no explicit specified
        if ( m_expSearchMaxEpochsBlend == -1 )
            m_expSearchMaxEpochsBlend = m_expSearchMaxEpochs;

        expSearchParams ( m_expSearchMinEpochs, m_expSearchMaxEpochsBlend, paramEpochs, accelerationEpochs, expInit );

        if ( m_enableDebug )
            cout<<endl<<endl<<"==================== auto-optimize ===================="<<endl<<endl;

        double error;
        while ( expSearchChangeParams() )
        {
            error = calcRMSEonBlend();
            expSearchCheckErr ( error );
        }
        if ( m_enableDebug )
            cout<<"expSearchErrorBest:"<<m_expSearchErrorBest<<"  error:"<<error<<endl;
    }
}

void AutomaticParameterTuner::expSearchReinitStepSize()
{
    if ( m_enableDebug )
        cout<<"Reinit exp search step to "<<m_expInit<<endl;
    m_expSearchDoubleParamPos = 0;
    m_expSearchIntParamPos = 0;
    m_expSearchErrorBest = 1e10;
    m_expSearchAcceleration = 0.8;
    m_expSearchAccelerationEpoch = 0;
    for ( int i=0;i<m_expSearchExponent.size();i++ )
        m_expSearchExponent[i] = m_expInit;
}

void AutomaticParameterTuner::expSearchSetEpochsToMinimizeBlend ( int epochs )
{
    m_expSearchMaxEpochsBlend = epochs;
}

double AutomaticParameterTuner::expSearchGetLowestError()
{
    return m_expSearchErrorBest;
}


// ====================== nelder-mead tuner ======================

void AutomaticParameterTuner::NelderMeadSearch ( int maxEpochs )
{
    int N = m_doubleParam.size() + m_intParam.size();
    vector<pair<double,vector<double> > > simplexPoints ( N+1 );

    // set the initial simplex
    for ( int i=0; i < N+1; i++ )
    {
        simplexPoints[i].second.resize ( N );
        for ( int j=0; j < m_doubleParam.size(); j++ )
            ( simplexPoints[i].second ) [j] = *m_doubleParam[j];
        for ( int j=0; j < m_intParam.size(); j++ )
            ( simplexPoints[i].second ) [j+m_doubleParam.size() ] = *m_intParam[j];

        if ( i < N )
            //(simplexPoints[i].second)[i] = (simplexPoints[i].second)[i] + fabs((simplexPoints[i].second)[i]) * 2.0;
            ( simplexPoints[i].second ) [i] = ( simplexPoints[i].second ) [i] + max ( 1.0,fabs ( ( simplexPoints[i].second ) [i] ) *2.0 );
    }

    for ( int i=0; i < N+1; i++ )
        simplexPoints[i].first = calcError ( simplexPoints[i].second );

    for ( int epoch =0; epoch < maxEpochs; epoch++ )
    {
        vector<double> center ( N );

        sort ( simplexPoints.begin(),simplexPoints.end() );
        calcCenter ( simplexPoints, center );
        plotParameters ( epoch, simplexPoints );

        // refelction
        vector<double> reflectionPoint ( N );
        double reflectionError = getReflectionPoint ( center,simplexPoints[N].second,reflectionPoint,1.0 );

        if ( ( reflectionError <= simplexPoints[N-1].first ) && ( reflectionError >= simplexPoints[0].first ) )
        {
            simplexPoints[N].second = reflectionPoint;
            simplexPoints[N].first = reflectionError;
            continue;
        }

        // expansion (the new point is the best so far)
        if ( reflectionError < simplexPoints[0].first )
        {
            vector<double> expansionPoint ( N );
            double expansionError = getReflectionPoint ( center,simplexPoints[N].second,expansionPoint,2.0 );

            if ( expansionError < reflectionError )
            {
                simplexPoints[N].second = expansionPoint;
                simplexPoints[N].first = expansionError;
                continue;
            }
            else
            {
                simplexPoints[N].second = reflectionPoint;
                simplexPoints[N].first = reflectionError;
                continue;
            }
        }

        // contraction (the new point is the worst so far)
        vector<double> contractionPoint ( N );
        double contractionError, refError;
        if ( reflectionError <= simplexPoints[N].first )
        {   // outside contraction
            refError = reflectionError;
            contractionError = getReflectionPoint ( center,reflectionPoint,contractionPoint,0.5 );
        }
        else
        {   // inside contraction
            refError = simplexPoints[N].first;
            contractionError = getReflectionPoint ( center,simplexPoints[N].second,contractionPoint,0.5 );
        }

        if ( contractionError <= refError )
        {
            simplexPoints[N].second = contractionPoint;
            simplexPoints[N].first = contractionError;
            continue;
        }

        // reduction
        for ( int j=1; j < N+1; j++ )
        {
            vector<double> reductionPoint ( N );
            double reductionError = getReflectionPoint ( center,simplexPoints[j].second, reductionPoint, 0.5 );
            simplexPoints[j].second = reductionPoint;
            simplexPoints[j].first = reductionError;
        }
    }

    sort ( simplexPoints.begin(),simplexPoints.end() );
    setCurrentParameter ( simplexPoints[0].second );
}

double AutomaticParameterTuner::calcError ( const vector<double> &paramVector )
{
    setCurrentParameter ( paramVector );

    return calcRMSEonProbe();
}

void AutomaticParameterTuner::setCurrentParameter ( const vector<double> &paramVector )
{
    for ( int i=0; i < m_doubleParam.size(); i++ )
    {
        double value = paramVector[i];

        if ( value <= m_doubleMin[i] )
            value = m_doubleMin[i];
        if ( value >= m_doubleMax[i] )
            value = m_doubleMax[i];

        *m_doubleParam[i] = value;
    }

    for ( int i=0; i < m_intParam.size(); i++ )
    {
        int value = ( int ) ( 0.5 + paramVector[i + m_doubleParam.size() ] );

        if ( value <= m_intMin[i] )
            value = m_intMin[i];
        if ( value >= m_intMax[i] )
            value = m_intMax[i];

        *m_intParam[i] = value;
    }
}

void AutomaticParameterTuner::calcCenter ( const vector<pair<double,vector<double> > > &simplexPoints, vector<double> &center )
{
    center.resize ( simplexPoints[0].second.size() );

    for ( int i=0; i < center.size(); i++ )
        center[i] = 0.0;

    for ( int i=0; i < simplexPoints.size()-1; i++ )
    {
        for ( int j=0; j < center.size(); j++ )
            center[j] += ( simplexPoints[i].second ) [j];
    }

    for ( int i=0; i < center.size(); i++ )
        center[i] /= ( double ) ( simplexPoints.size()-1 );
}

double AutomaticParameterTuner::getReflectionPoint ( const vector<double> &center, const vector<double> &worstPoint, vector<double> &reflectionPoint, double alpha )
{
    int N = center.size();
    reflectionPoint.resize ( N );

    for ( int i=0; i < N; i++ )
        reflectionPoint[i] = center[i] + alpha* ( center[i] - worstPoint[i] );

    return calcError ( reflectionPoint );
}

void AutomaticParameterTuner::plotParameters ( int epoch, const vector<pair<double,vector<double> > > &simplexPoints )
{
    static time_t t0 = time ( 0 );

    if ( m_enableDebug )
        cout<<epoch<<". | error: ["<<simplexPoints[0].first<<"-"<<simplexPoints[simplexPoints.size()-1].first<<"] ";

    for ( int i=0; i < m_doubleParam.size(); i++ )
    {
        double min,max;
        calcMinMax ( simplexPoints,i,min,max );
        if ( m_enableDebug )
            cout<<" | "<<m_doubleName[i]<<": ["<<min<<" - "<<max<<"]";
    }

    for ( int i=0; i < m_intParam.size(); i++ )
    {
        int param = i+m_doubleParam.size();
        double min,max;
        calcMinMax ( simplexPoints,param,min,max );
        if ( m_enableDebug )
            cout<<" | "<<m_intName[i]<<": ["<<min<<" - "<<max<<"]";
    }

    time_t t1 = time ( 0 );
    if ( m_enableDebug )
        cout<<" | "<<t1-t0<<"[s]"<<endl;
    t0=t1;
}

void AutomaticParameterTuner::calcMinMax ( const vector<pair<double,vector<double> > > &simplexPoints, int paramNr, double &min, double &max )
{
    min = DBL_MAX;
    max = DBL_MIN;

    for ( int i=0; i < simplexPoints.size(); i++ )
    {
        double value = simplexPoints[i].second.at ( paramNr );

        if ( value < min )
            min = value;
        if ( value > max )
            max = value;
    }
}