import os
from datetime import datetime

from garcon import task
import sklearn
from sklearn.ensemble import RandomForestClassifier

import utils
import utils.s3 as s3
import utils.Google.Docs.read_sheet as gdoc
import query
import config

@task.decorate(timeout=60)
def bootstrap(activity):
    # set up the dates!
    run_date = datetime.now().strftime('%Y-%m-%d')
    year = datetime.now().strftime('%Y')
    month = datetime.now().strftime('%m')

    # backfill_date = datetime(2017,2,28)
    # run_date = backfill_date.strftime('%Y-%m-%d')
    # year = backfill_date.strftime('%Y')
    # month = backfill_date.strftime('%m')

    # Check if there's a recent model.
    models = s3.ls(config.latest_model, include_bucket=True)

    if len(models) == 0:
        print("No model present!")
        return {'stop': True}

    latest_model = models[0]
    next_model = config.next_model.format(run_date=run_date)
    
    # will old models retire.
    model_archive = latest_model.replace(
        'models/latest/', 'models/archive/{}/{}/'.format(year, month))

    # S3 Keys
    s3_filedrop = config.s3_drop_path
    s3_archive_path = config.s3_archive_path.format(
        run_date=run_date, year=year, month=month)
    s3_processed_path = config.s3_processed_path.format(
        run_date=run_date, year=year, month=month)
    s3_machine_v_human = config.s3_machine_v_human.format(
        run_date=run_date, year=year, month=month)
    s3_predictions = config.s3_predictions.format(
        run_date=run_date, year=year, month=month)
    
    # Get the file targets
    regEx = s3_filedrop + '/*.xlsx'
    new_files = s3.ls(regEx , include_bucket=True)

    if len(new_files) == 0:
        return {'stop': True}

    # where raw files will be stored.
    archives = [f.replace(s3_filedrop, s3_archive_path) for f in new_files]

    # where preprocessed files will be stored.
    processed = [f.replace(s3_filedrop, s3_processed_path)\
                  .replace('.xlsx', '.tsv.gz') \
                  for f in new_files]
                  
    context = dict(
        run_date=run_date,
        s3_filedrop=new_files,
        s3_archive=archives,
        s3_processed=processed,
        s3_machine_v_human=s3_machine_v_human,
        s3_predictions=s3_predictions,
        latest_model=latest_model,
        next_model=next_model,
        model_archive=model_archive)

    return context

@task.decorate(timeout=60)       
def archive_raw(activity, s3_archive, s3_filedrop):
    
    for f1, f2 in zip(s3_filedrop, s3_archive):        
        s3.mv(old_path=f1, new_path=f2)


@task.decorate(timeout=60)       
def process_excel(activity, s3_archive, s3_processed, run_date):
    
    for f1, f2 in zip(s3_archive, s3_processed):
        utils.read_excel(f1, f2, run_date)

@task.decorate(timeout=60 * 4)
def upload_stg(activity, s3_processed, run_date):
    sf = utils.sf

    for f in s3_processed:
        # insert to staging table
        sf.create_table(f, config.sf_results_stg, config.sf_fmt)
        resp = sf.s3_2_table(f, config.sf_results_stg, config.sf_fmt)

        if resp.get('code') == 200:
            # remove duplicate UPCs.
            resp = sf.q(query.delete_dupes)

            if resp.get('code') == 200:
                # insert new results with date.
                resp = sf.q(query.insert_results.format(run_date=run_date))
                
                if resp.get('code') == 200:
                    sf.drop(config.sf_results_stg)
        else:
            print("Nope Nope Nope")

@task.decorate(timeout=60 * 10)
def evaluate_new_data(activity, latest_model, s3_machine_v_human):
    '''
    Gets probability scores for new UPCs.
    Confusion matrix here?
    '''
    print("\nRunning new data through existing Random Forest (RF) model.")
    df = utils.get_new()

    clf = s3.load_clf(latest_model)
    # print('\n{}\nparameterized as follows:\n{}'.format(latest_model, clf))

    if isinstance(clf, sklearn.ensemble.forest.RandomForestClassifier):
        X, y = utils.sanitize(df), df[config.target]

        df[config.prob] = clf.predict_proba(X)[:, 1]
        utils.batch_update_probs(df, s3_path=s3_machine_v_human) 

        score = clf.score(X, y)
        print('\nAccuracy for new points {0:.2f}%'.format(score * 100))

    else:
        print("model not imported correctly.")

    return {'score': score}


@task.decorate(timeout=60 * 6)
def refit_model(activity, run_date, latest_model, next_model, model_archive):
    '''
    Pull data
    instantiate a new model
    fit new model to training set (X_train, y)train
    upload new weights
    '''
    print('\nRefreshing Model...')

    X_train, X_test, y_train, y_test = utils.get_train_test_split()

    print('\nFitting new RF Model with training set(n={})'.format(len(X_train)))
    clf = RandomForestClassifier(random_state=42, n_jobs=-1, n_estimators=150)

    clf.fit(utils.sanitize(X_train), y_train)
    # print('Training complete!\n{}'.format(clf))
    
    print('\nWith Feature importance:')    
    utils.feature_importance(utils.sanitize(X_train), clf)

    # updating weights...
    utils.upload_weights(clf, latest_model, next_model, model_archive)
    
    print('\nEvaluating new RF Model with test set(n={})\n'.format(len(X_test)))
    resp = utils.upload_meta(clf, X_train, X_test, y_train, y_test, run_date)
    
    if resp['code'] == 200:
        return

@task.decorate(timeout=60 * 14)
def predict_catalog(activity, next_model, s3_predictions):
    '''
    What to do with the already reviewed ones?
    '''
    print('\nGetting delivery probability distribution '
            'for catalog using new RF model.')

    clf = s3.load_clf(next_model)
    df = utils.get_catalog()

    # make prediction on unreviewed catalog
    cat = df[ df[config.target].isnull() ]

    print('\n{} releases being run through the RF model.'.format(len(cat)))
    cat[config.prob] = clf.predict_proba(utils.sanitize(cat))[:, 1]

    cat.to_csv('catalog_not_reviewed.tsv', index=False, sep='\t')

    df.set_index('UPC', inplace=True)
    df.update(cat.set_index('UPC'))
    df.reset_index(inplace=True)

    df.to_csv('catalog_all.tsv', index=False, sep='\t')

    # this change has been messing things up...
    genre_cols = [c for c in df.columns if 'GENREID_' in c]
    df['GENREID'] = utils.reverse_dummy(df[genre_cols])
    df['GENRENAME'] = df['GENREID'].map(utils.get_genremap())

    df = df[[col for col in df.columns if col not in genre_cols]]

    print('Uploading new predictions to {}'.format(s3_predictions))
    s3.to_csv(df, s3_predictions, sep='\t', 
              compression='gzip', index=False)

    sf = utils.sf

    # get tables.
    sf_predictions = config.sf_predictions
    sf_predictions_stg = sf_predictions + '_STG'
    
    sf.create_table(s3_predictions,
                    sf_predictions_stg,
                    config.sf_fmt,
                    dtype= {'UPC': 'O', 'NOTE': 'O'},
                    debug= False)

    sf.s3_2_table(s3_predictions,
                  sf_predictions_stg,
                  config.sf_fmt)

    resp = sf.q('CREATE OR REPLACE TABLE {} CLONE {}'.format(
        sf_predictions, sf_predictions_stg))
    # resp = sf.insert(sf_predictions, sf_predictions_stg)
    
    if resp['code'] == 200:
        sf.drop(sf_predictions_stg)
        print('Predictions updated on Looker!')

@task.decorate(timeout=60)
def receipt(activity):
    utils.email(config.email_recipients,
                config.email_header,
                config.email_body.format(1,2,3)) # replace 1,2,3 with links!