from eventlet import *
import time
import logging
import boto
import os
import gzip
import csv
import string
import json
import sys
import pprint

import dimensions

config = json.loads(open('../config/config.json').read())

AWS_ACCESS_KEY_ID = str(config['aws']['accessKeyId'])
AWS_SECRET_ACCESS_KEY = str(config['aws']['secretAccessKey'])
AWS_REGION = str(config['aws']['region'])


logging.basicConfig(filename="s3_download.log", level=logging.INFO)
dimension_dicts = dimensions.dimension_info()

def is_number(s):
    try:
        int(s)
        return True
    except ValueError:
        return False


def process_file(key_name, s3_bucket, output_file_name, job_type):

    key = s3_bucket.get_key(key_name)


    final_array = {}
    res = key.get_contents_to_filename(key.name)
    dim_dict = dimension_dicts[job_type]
    #for line in csv.reader(open(fname), delimiter=' '):
    #file_length = 0
    #with gzip.open(key.name) as f:
    #    for _ in f:
    #        file_length += 1
    #sys.exit(str(file_length))
    loop_line_count = 0
    for line_array in csv.reader(gzip.open(key.name), delimiter='\t', quotechar='"', quoting=csv.QUOTE_MINIMAL, escapechar='\\'):
        #line_array = string.split(line, sep="\t")
        loop_line_count += 1
        for index, column in dimension_dicts[job_type]['columns'].iteritems():
            try:
                insertion_value = line_array[index]
            except IndexError:
                pprint.pprint(line_array)
                print str(index)
                pprint.pprint(key.name)
            if is_number(insertion_value) == True:
                final_array[column] = int(insertion_value)
            else:
                if line_array[index] == 'Y':
                    final_array[column] = "True"
                elif line_array[index] == 'N':
                    final_array[column] = "False"
                else:
                    final_array[column] = insertion_value
            #pprint.pprint(final_array)
            #pprint.pprint(json.dumps(final_array))
        # final_array[dimension_dicts['artist'][0]] = int(line_array[0])
        # final_array[dimension_dicts['artist'][1]] = line_array[1]
        # final_array[dimension_dicts['artist'][2]] = int(line_array[2])
        #print  #indent
        else:
            output_file = open(output_file_name, 'a')
            output_file.write(json.dumps(final_array) + "\n")

Bucketname = 'prod-dimension-tables' 
logging.info('connecting..')

conn = boto.s3.connect_to_region(AWS_REGION,
       aws_access_key_id=AWS_ACCESS_KEY_ID,
       aws_secret_access_key=AWS_SECRET_ACCESS_KEY
       )
logging.info('traverse to bucket:' + Bucketname)
bucket = conn.get_bucket(Bucketname)
logging.info('in the bucket')

logging.info("Spawn some threads.")

pool = GreenPool(size=20)

logging.info("Saving files from bucket...")


#@TODO have the date and the bucket location be dynamic, or pull from message
bucket_prefix = "2014-04-25 01:00:16/"
#@TODO have the dimension table schema definitions come from s3 as well

full_start_time = time.time()
#for job_type, dimension_table in dimension_tables:
single_dimension_start_time = time.time()

output_file_name = sys.argv[1]
export_job_type = sys.argv[2]
clean_out_file = open(output_file_name, 'w+')
clean_out_file.close()

directory = bucket_prefix + dimension_dicts[export_job_type]['table'] + "/"
#sys.exit(directory);


if not os.path.exists(directory):
    os.makedirs(directory)
for key in bucket.list(prefix=directory):
    logging.info(key.key)
    # res = key.get_contents_to_filename(key.name)
    # key = bucket.get_key(key.name)
    # try:
    #     res = key.get_contents_to_filename(key.name)
    # except:
    #     logging.info(key.name+":"+"FAILED")
    pool.spawn_n(process_file, key.key, bucket , output_file_name, export_job_type)
    pool.waitall()
logging.info(export_job_type + " loading took " + str(time.time() - single_dimension_start_time) + " seconds")

logging.info("full job  took " + str(time.time() - full_start_time) + " seconds")