## Get DT.country earlier... it gets used in subsequent functions
options("snowflake_inuse")
DT.country <- get_dim_country(refresh=TRUE, assign=FALSE)
setkeyIfNot(DT.country, country_code, organize=TRUE, warnForColNameInEnv=FALSE)

## TODO:  Confirm these dont fuck up format
## Ignoring these for now. 
files_pre2013 <- dir(data.p("ftp_sync", "old_files", "processed"), full=TRUE)

## Pull the files from disk
files.ftp_sync           <- dir(data.p("ftp_sync")             , full=TRUE)
files.ftp_sync.old_files <- dir(data.p("ftp_sync", "old_files"), full=TRUE)

## Remove files from the OLD folder which are already in the new folder
files.ftp_sync.old_files %<>% { . [basename(.) %ni% basename(files.ftp_sync)]}

## There should be no intersect
stopifnot(0 == length(intersect(basename(files.ftp_sync.old_files), basename(files.ftp_sync))))

files_on_disk <- c(files.ftp_sync.old_files, files.ftp_sync)

if (any(wh.missing <- path.expand(unlist(files.brought)) %ni% files_on_disk)) {
  warning(warningCols("Some files just brought over are missing from files_on_disk: ", unlist(files.brought)[wh.missing], cols=1))
}

## No idea what these are, but these seem to all be duplicates
{
  tmp_files.ftp_sync.the_orchard <- dir(data.p("ftp_sync", "theorchard"), full=TRUE)
  ## remove the file lovey.txt
  tmp_files.ftp_sync.the_orchard %<>% {.[basename(.) != "lovey.txt"]}
  stopifnot(basename(tmp_files.ftp_sync.the_orchard) %in% basename(files_on_disk))

  ## If the above fails
  if (FALSE) 
  {
    basename(tmp_files.ftp_sync.the_orchard)[!(basename(tmp_files.ftp_sync.the_orchard) %in% basename(files_on_disk))]
    tmp_files.ftp_sync.the_orchard[[1]] %>% readLines
  }
}

## File names and path
fullFilePaths <- unlist(files_on_disk, use.names=FALSE) %>% {.[!isdir(.)]}

## Some files (so far only of type 'invoice') had occasional updates/fixes sent
## In those cases, we need to grab the latest such file.
## I am assuming that the 4-digit number at the end is what indicates this. 
## I could be wrong
# pat.updates <- "_\\d{4}\\."
# pat.updates <- sprintf("(_\\d{4})+\\..*$")
# removeText(sprintf("(%s)+\\..*$", removeText("\\\\\\.$", pat.updates))
# tmp_files_with_updates <- extract(pat.updates, basename(fullFilePaths))
# base_base <- removeText(sprintf("(%s)+\\..*$", removeText("\\\\\\.$", pat.updates)), tmp_files_with_updates)
# tmp_files_keeping <- tapply(X=tmp_files_with_updates, INDEX=base_base, FUN=max)
# tmp_files_dropping <- setdiff(tmp_files_with_updates, tmp_files_keeping)
# fullFilePaths %<>% {.[basename(.) %ni% tmp_files_dropping]}

fullFilePaths <- fullFilePaths [order(dateFromSpotifyFile(fullFilePaths), na.last=FALSE)]
classAppend_(fullFilePaths, "datestamped.filenames")

## set the names of fullFilePaths to the basenamse of the files
setattr(fullFilePaths, "names", basename(fullFilePaths))

## Drop duplicate files from fullFilePaths, so that they add up
## WRONG!  -- dont extract this pattern; instead combine these, they are additional data, not replacement data
## WRONG!  fullFilePaths %<>% removeDuplicateFilesFromFileList.byName(pat.to_strip="(_\\d{4})+(?=\\.)", perl=TRUE, time_name_conflict_use="name")
fullFilePaths %<>% removeDuplicateFilesFromFileList.byName(pat.to_strip=NULL, time_name_conflict_use="name")


xls.pat <- "\\.xls.*?$"
pdf.pat <- "\\.pdf$"
pdfFiles      <- extract(pdf.pat,    fullFilePaths)
downloadFiles <- extract("download", fullFilePaths)
linkFiles     <- extract("Shortcut\\.lnk$", fullFilePaths)

## THESE ARE THE MAIN FILES STILL BEING SENT
breakageFiles <- extract("breakage", fullFilePaths)                                   #  %>% removeDuplicateFilesFromFileList.byName
dataFiles     <- extract("[-_]data", fullFilePaths)                                   #  %>% removeDuplicateFilesFromFileList.byName
invoiceFiles  <- extract("invoice",  fullFilePaths[fullFilePaths %ni% pdfFiles])      #  %>% removeDuplicateFilesFromFileList.byName
legendFiles   <- extract("legend",   fullFilePaths)                                   #  %>% removeDuplicateFilesFromFileList.byName
revshareFiles <- extract("revshare", fullFilePaths[fullFilePaths %ni% downloadFiles]) #  %>% removeDuplicateFilesFromFileList.byName
cohortFiles   <- extract("cohort",   fullFilePaths)                                   #  %>% removeDuplicateFilesFromFileList.byName
xlsFiles      <- extract(xls.pat,    fullFilePaths[fullFilePaths %ni% cohortFiles])   #  %>% removeDuplicateFilesFromFileList.byName

## eg for Julyl 2015 we received
##     * partner_intake_and_monthly_cohort_average_20150731.xlsx
##     * spotify-breakage-for-theorchard-20157.txt
##     * spotify-data-for-theorchard-20157.txt
##     * spotify-invoice-for-theorchard-20157_6195.txt
##     * spotify-legend-for-theorchard-20157.txt
##     * spotify-revshare-for-theorchard-20157.txt
##
## The 'data' file itself contains sevearl reports: 


classAppend_(revshareFiles, "datestamped.filenames")

# ## Split out the download files 
# revshareFiles <- revshareFiles[revshareFiles %ni% downloadFiles]

## The above should cover all files
allFileGroups <- c(revshareFiles, downloadFiles, breakageFiles, dataFiles, legendFiles, cohortFiles, xlsFiles, pdfFiles, invoiceFiles, linkFiles)
classAppend_(allFileGroups, "datestamped.filenames")

if (length(fullFilePaths) != length(allFileGroups))
  stop(sprintf("Differring lengths for fullFilePaths (%i files) and allFileGroups (%i files). \nHint: allFileGroups is meant to be the collection of all of the files broken out from fullFilePaths\nIf it has less file-length you missed something;  if it has more, there might be a bug.\n\nCheck    setdiff(fullFilePaths, allFileGroups)", length(fullFilePaths), length(allFileGroups)))
## CONFIRM: we have exactly all of the files
stopifnot(length(fullFilePaths) == length(allFileGroups))
stopifnot(!length(setdiff(fullFilePaths, allFileGroups)), !length(setdiff(allFileGroups, fullFilePaths)))
stopifnot(identical(sort(fullFilePaths), sort(allFileGroups)))

## CONFIRM: No Duplicate files. Check twice,
stopifnot(sapply(allFileGroups, function(x) anyDuplicated(names(x))) == 0)
stopifnot(sapply(allFileGroups, function(x) anyDuplicated(basename(x))) == 0)

## Insert a line break into the output
if (length(xlsFiles) || length(pdfFiles))
{
  cat("\n\n", pasteR(66), sep="")
  if (length(xlsFiles)) {
    basenames.xlsFiles <- gsub("(_| )*(\\d{5,8}|\\d{4}-\\d{1,2}(-\\d{1,2})?)$", "", gsub(xls.pat, "", names(xlsFiles), ignore.case=TRUE))
    dates.xlsFiles <- dateFromSpotifyFile(xlsFiles)
    cat("\nThe following xls files exist: \n")
    print(data.table(date=dates.xlsFiles, name=basenames.xlsFiles)[, list(minDate=min(date), maxDate=max(date)), by=list(`File Basename`=name)])
  }
  if (length(pdfFiles)) {
    basenames.pdfFiles <- gsub("(_| )*(\\d{5,8}|\\d{4}-\\d{1,2}(-\\d{1,2})?)$", "", gsub(pdf.pat, "", names(pdfFiles), ignore.case=TRUE))
    dates.pdfFiles <- dateFromSpotifyFile(pdfFiles)
    cat("\nThe following pdf files exist: \n")
    print(data.table(date=dates.pdfFiles, name=basenames.pdfFiles)[, list(minDate=min(date), maxDate=max(date)), by=list(`File Basename`=name)])
  }
  cat(pasteR(66), "\n\n", sep="")
}


## Legend Table
## We can ignore the datestamp in the file, and instead take unique() of the rbind of all the dictionaries
DT.spotify_product_dict <- unique(setkey(rbindlist(lapply(legendFiles, fread))))
simpleClean_of_spotify_DT_(DT.spotify_product_dict, convertFactors=FALSE)

## Breakage DTs
DT.breakage  <- readSpotifyFiles(breakageFiles, append.content_owner=TRUE, append.file_name=FALSE, verbose=FALSE)

## downloads DT
## NOTE: downloads is an old concept for spotify and no recent files have this.  As of 2016, it appears these files have been archived / removed. 
DT.downloads <- if (length(downloadFiles))  readSpotifyFiles(downloadFiles, append.content_owner=TRUE, append.file_name=FALSE, verbose=FALSE)

## Revenue Share
DT.revshare  <- readSpotifyFiles(revshareFiles, append.content_owner=TRUE, append.file_name=FALSE, verbose=TRUE)
cleanRevshare_(DT.revshare)

## list of "DATA" DTs
## This whole section can be cleaner. But it works
{
  # rm(list.DT.datainfo, DT.Breakage_Rate)
  list.DT.datainfo <- readSpotifyDataFiles(dataFiles)
  nms.DT.data <- gsub(" ", "_", names(list.DT.datainfo))
  data.DTs <- setNames(nm=nms.DT.data, sprintf("DT.%s", nms.DT.data))

  ## Iterate over each DT, clean it, then assign it
  for (.nm in nms.DT.data) {
    cat("Creating DT.", .nm, " from datainfo files\n", sep="")
    DT <- copy(list.DT.datainfo[[.nm]])
      
    simpleClean_of_spotify_DT_(DT, remove_file_metadata=(.nm != "MISSING_DATA"), convertFactors=TRUE )

    assign(data.DTs[[.nm]], DT, envir=globalenv())
    rm(DT, .nm)
  }
}

## ------------------------------------------------------------------ ##

## Confirm all of the countries in the data are captured in our internal dim_country, as represented in DT.country
countries_in_spotify_data <- unique(unlist(gapply(data.DTs, "[[", "country_code")))

## Confirm all are in Spotify
if (any(countries_in_spotify_data %ni% DT.country$country_code))
  warning("Some countries in Spotify data are NOT in our dim_country data, namely: ", pasteQand(setdiff(countries_in_spotify_data, DT.country$country_code)))


print(saveImageTo(subProj="After Part 2"))