import os
from io import FileIO, StringIO
import httplib2
import datetime

from apiclient.http import MediaIoBaseDownload
from apiclient.discovery import build
from apiclient.errors import HttpError
from oauth2client.client import flow_from_clientsecrets
from oauth2client.file import Storage
from oauth2client.tools import argparser, run_flow

from Snowflake import sfq
import s3
import config

##############################################################################
########################### Initiate Context #################################
##############################################################################
def build_context(job_raw, content_owner, starttime):
    """
    job_raw comes from job_metas = see_jobs(content_owner)
    checks to see if a s3 file (the target) exists for a given report type / CMS/ date.
    a response body is returned with metadata about the report.
    """

    enddate = startdate
    job_name = job_raw['name'][:-11].replace('_','')
    
    # normalize names for v1.1 reports
    job_name = v1_1_reports.get(job_name, job_name)
    
    # get all reports from a given job and time range.
    target_report = youtube_reporting.jobs().reports().list(
	    onBehalfOfContentOwner=content_owner,
	    jobId=job_raw['id'],
	    startTimeAtOrAfter = starttime,
	    startTimeBefore = endtime
    ).execute()['reports']
    
    if target_report:
        '''
        Build several strings of file paths for files that will
        touch a local stage, be stored in AWS S3, and eventually
        populate a table in Snowflake.
        '''
        # Where the file will live in s3
        target = os.path.join(
            S3_PATH, job_name, LAST_WEEKS_YEAR, LAST_WEEKS_MONTH
            ,'R' + LAST_WEEK, CMS_DICT.get(content_owner), 'DL' + TODAY, 
            job_name + '_R{}_DL{}.csv.gz'.format(LAST_WEEK, TODAY)
        )
        
        # Same as target w/out download date (to check it exists)
        target_pattern = os.path.join(
            S3_PATH, job_name, LAST_WEEKS_YEAR, LAST_WEEKS_MONTH,
            'R' + LAST_WEEK, CMS_DICT.get(content_owner),'*', 
            job_name + '_R{}_DL{}.csv.gz'.format(LAST_WEEK, TODAY)
        )

        # Local staging path for file.
        target_lite = os.path.join(
            STAGE, CMS_DICT.get(content_owner),
            target.split('/')[-1].replace('.gz','')
        )
        
        # Snowflake table where data must end up.
        target_payload = '.'.join([PROD_DB, PROD_SCHEMA, job_name])
        
        responseBody = {
            "s3Target":         target,
            "localDestination": target_lite,
            "payload":          target_payload,
            "downloadUrl":      all_reports[0]['downloadUrl']
        }

      
    else:
        print("{} not available for {}".format(job_raw, starttime))
            
    return responseBody

##############################################################################
########################### YOUTUBE API CALLS ################################
##############################################################################

def get_authenticated_services(SCOPES):
    """
    Before using the Youtube API, we must be be authenticated.
    The authentication process is as follows:
    1. request access token using your CLIENT_SECRETS_FILE,
       to perfom tasks in YOUTUBE_SCOPES.
    2. store access token in local directory.
    3. check for valid credentials from the access token.
    4. Based on the actions wished to perform, a different youtube object
       is built for reporting or querying.
    """
    # 1
    flow = flow_from_clientsecrets(
        CLIENT_SECRETS_FILE,
        scope = " ".join(SCOPES),
        message = MISSING_CLIENT_SECRETS_MESSAGE
        )
 
    # 2
    storage = Storage(TOKEN)
    credentials = storage.get()

    # 3
    if credentials is None or credentials.invalid:
        credentials = run_flow(
            flow, storage, argparser.parse_args(args=[])
        )

    http = credentials.authorize(httplib2.Http())
  
    # 4
    youtube_reports = build(YOUTUBE_REPORTING_API_SERVICE_NAME, 
                            YOUTUBE_REPORTING_API_VERSION,
                            http = http)
    return youtube_reports

def see_jobs(content_owner):
    """
    Shows reporting jobs for each content owner in a df.
    """
    youtube_reporting = get_authenticated_services(YOUTUBE_REPORT_SCOPES)
    
    return youtube_reporting.jobs().list(
        onBehalfOfContentOwner=content_owner
    ).execute()['jobs']

##############################################################################
########################### MOVING DATA'ROUND ################################
##############################################################################

def download_report(local_file, url):
    """
    Creates a youtube_reporting token and a media HTTP request.
    Sets the object's uri to the given url
    Creates file object from Filename to write contents of HTTP request.
    Downloads the request chunkwise to file, returns status.
    """
    youtube_reporting = get_authenticated_services(
        YOUTUBE_REPORT_SCOPES
        )
    
    request = youtube_reporting.media().download(resourceName="")
    request.uri = url

    with FileIO(local_file, mode='wb') as fh:
        downloader = MediaIoBaseDownload(fh, request, chunksize=2e8)

        done = False
        while done is False:
            status, done = downloader.next_chunk(num_retries=10)
        
        # gzip it
        gzip(filename)
        
        return filename

def upload_report(local_file,s3_path):
	"""
	Uploads a local file to s3.
	removes the local file from disk.
	"""
	resp = s3.disk_2_s3(local_file,s3_path)
	os.remove(local_file)
	return resp

def insert_to_table(s3_path,table,fmt):
    prod = table
    dev = table + '_dev'
    
    sfq("CREATE or REPLACE TABLE {} LIKE {}".format(dev,prod))
    
    resp1 = s3_2_table(s3_path,dev,fmt)
    resp2 = sfq("INSERT INTO {} SELECT * FROM {}".format(prod,dev))
    
    sfq("DROP TABLE {}".format(dev))
    
    return({"ingestion":resp1,"insert":resp2})

##############################################################################
############################# TARGET PRACTICE ################################
##############################################################################

def exists_in_s3(target_pattern):
    """
    Checks if an s3 pattern (or file) exists.
    If s3.look (essentially an s3-glob) returns any files, the pattern exists.
    """
    exists = True
    
    hit = s3.look(target_pattern)
    if not hit:
        exists = False
    
    return exists

def exists_in_table(table, content_owner, starttime):
    """
    Reads logfile into a Pandas dataframe via log_dataframe.
    List comp through report_date -- checks that the current download's date
    is not in that list.
    returns True is the date is already in the table, else False.
    """
    delivered = False

    query = """
    SELECT * FROM {} 
    WHERE CONTENT_OWNER = {} 
    AND DATE = {}""".format(
    	table,
    	content_owner,
    	starttime    
    )

    df = sfq(query)

    if isinstance(df,pd.DataFrame):
    	if not df.empty:
            delivered = True
    
    return delivered

def exists_upstream(content_owner, jobid, startime, endtime):
	"""
	For a given job, see if there are any jobs with the startdate.
	"""
	exists = False

	hit = youtube_reporting.jobs().reports().list(
	    onBehalfOfContentOwner=content_owner,
	    jobId=jobid,
	    startTimeAtOrAfter = starttime,
	    startTimeBefore = endtime
    ).execute()['reports']

	if hit:
		exists = True

	return exists

##############################################################################
########################### GENERAL FUNCTIONS ################################
##############################################################################

def gzip(filename):
    """
    Uses subprocess' Popen to use the PIGZ program to compress the filename.
    The old file.csv is replaced by file.csv.gz
    """
    process = Popen(['/home/lyin/.linuxbrew/bin/pigz','--best',filename],
                    stdout=PIPE, stderr=PIPE)
    
    stdout, stderr = process.communicate()