# 01 Retrieve Files From FTP.r
setScience("Spotify_Accounting_ETL", create=TRUE, subl=FALSE, load=FALSE)

lib(RCurl, quietly=TRUE)
lib(bit64, quietly=TRUE)

## 2015-12-05:  setSnowflake_inuse issue
## snowflake in use is getting overwridden somewhere in set science
## For now check for 'wh' and 'dbname'
## TODO:  Fix setScience to no longer default to redshift
if (exists("wh") && exists("dbname") && .Pfm != "Darwin")
  setSnowflake(wh=wh, dbname=dbname)


ftp     = "ftp://ftp.theorchard.com"
local.basefolder = data.p("ftp_sync")
user = "spotify"
password = "Sp0t1fY"
verbose.curl = FALSE
verbose=TRUE


## --------------------------------- ##
##             EXECUTE               ##
## --------------------------------- ##

userpwd <- sprintf("%s:%s", user, password)

## bring over the files, first checking what is in the dir, to see which files are new
files.already_present <- dir(local.basefolder, recursive=TRUE, full=TRUE)
files.brought <- dlFilesFromFTP (ftp=ftp, local.basefolder=local.basefolder, recursive=TRUE, pattern.including=NULL, pattern.excluding="(spotify-(album|track)|test_ftp)", userpwd=userpwd, maxSize=1.05e8, verbose=verbose, verbose.curl=verbose.curl)

## Take set diff twice, with and without path.expand
files.newly_brought <- setdiff(unlist(files.brought, use.names=FALSE), files.already_present)
files.newly_brought <- files.newly_brought[path.expand(files.newly_brought) %ni% path.expand(files.already_present)]


## Indicate how many new files, with date info of what was brought ##
## --------------------------------------------------------------- ##
dates_of_new_files <- table(monthFloor(dateFromSpotifyFile(files.newly_brought)))
msg.date_range <- {
  if (length(dates_of_new_files) == 1)
      paste0("All files are for the month of ", names(dates_of_new_files))
  else if (length(dates_of_new_files) > 1)
      paste0("Files are from the following months:\n", pasteC(sprintf("%s :: % 2i files%13s", names(dates_of_new_files), dates_of_new_files, "."), C="\n"))
}
verboseMsg(verbose, boxit(paste(ifelse(!length(files.newly_brought), "No new files have been brought over", sprintf("%i new files have been brought over", length(files.newly_brought))), "from the ftp server", "\n", msg.date_range)), time=FALSE, func="message")