setScience("Spotify_API_V2", create=TRUE, subl=FALSE, load=FALSE)
lib(RCurl)
lib(rjson)
lib(doMC)


registerDoMC(ifelse(getPfm() == "Darwin", 4, 24))

## Modifiable Parameters
assignIfNotExist(TEST_RUN,              FALSE)
assignIfNotExist(minDate.using,         as.Date("2014-08-01"))
assignIfNotExist(processed.subfolder,   "processed") ## Where processed files will be stored, relative to the file
assignIfNotExist(verbose,               TRUE)

## In case user forgot to call as.Date() when assigned
if (is.character(minDate.using))
  minDate.using <- as.Date(minDate.using, origin=.origin)


## DEBUGGING
stopifnot(identical(minDate.using, as.Date("2014-10-01")))

## Not iterators, used for building URL
base.url <- "https://ws.spotify.com/analytics/api/theorchard"
token <- getNewSpotifyToken()
token.str <- sprintf("?oauth_token=%s", token)


## DATES
dates.using <- seq(Sys.Date()-1, minDate.using, by="-1 day")

## DATA TYPES
datatypes.using <- c("streams", "tracks", "users")  # "social" ?

## COUNTRY
country_codes.spotify <- getUniqueCountryCodes(refresh=FALSE)
country_codes.other  <- c()  ## TODO 
country_codes.using <- sort(unique(c(country_codes.spotify, country_codes.other)))

if (TEST_RUN) {
  dates.using <- sort(sample(head(dates.using, 25), 2))
  country_codes.using <- c("US", "SE")
}

## Check country has data 
wh.ct_is_live <- sapply(as.path("https://ws.spotify.com/analytics/api/theorchard/streams/2014/10/01/", country_codes.using, paste0("/?oauth_token=", token)), url.exists)
country_codes.NOT_used <- country_codes.using[!wh.ct_is_live]
country_codes.using <- country_codes.using[wh.ct_is_live]

if (!TEST_RUN)
  jesusForData(country_codes.spotify, country_codes.using, country_codes.NOT_used)

## CREATE folders, by file type
sapply(data.p(datatypes.using), dir.create, showWarnings=FALSE)

## Create country folders, just in streams
sapply(data.p("streams", country_codes.using), dir.create, showWarnings=FALSE)

if (isExpired(token))
  token <- getNewSpotifyToken()

### --- unique to streams ------- ##
{
cat("Iterating over ", length(country_codes.using), " countries:\n   ", commaSep(country_codes.using), "\n")
cat("Iterating over ", length(dates.using), " dates:\n   From ", format(min(dates.using), "%Y-%m-%d"), " to ", format(max(dates.using), "%Y-%m-%d"),  "\n")

  ret_streams.ll <- vector("list", length=length(country_codes.using) * length(dates.using))
  i <- 0

  for(country in country_codes.using)
    for(date in dates.using)
     {
          i <- i + 1

          datatype <- "streams"

          date <- as.Date(date, origin=.origin)
          date.str  <- format(date, "%Y%m%d")

          ## Create URL & file
          URL <- as.path(base.url, datatype, format(date, "%Y/%m/%d"), country, token.str)

          file.out <- data.p(datatype, country, sprintf("%s_%s_%s", datatype, country, date.str), ext="gz")

          ## For output
          url.out <- gsub(escapeRegEx(token.str), "", URL)

          if (fileDownloaded(file.out, processed.subfolder=processed.subfolder, minSize=0, check=TRUE)) {
            results="previously downloaded"
          } else {
            if (verbose)
              cat(file.out, "\n")
            downloadBinFile(URL, file.out)
            results <- if (fileDownloaded(file.out, processed.subfolder=processed.subfolder, minSize=0, check=TRUE)) "success" else "ERROR"
          }

          ret_streams.ll[[i]] <- c(date=date.str, datatype=datatype, country=country, results=results, url=url.out, file=file.out)
       }
    ret_streams <- do.call(rbind, ret_streams.ll)



    ret_rest.ll <- vector("list", length=length(country_codes.using) * length(dates.using))
    i <- 0

     for(datatype in setdiff(datatypes.using, "streams"))
      for(date in dates.using) 
       {
          i <- i + 1

          date <- as.Date(date, origin=.origin)
          date.str  <- format(date, "%Y%m%d")

          ## Create URL & file
          URL <- as.path(base.url, datatype, format(date, "%Y/%m/%d"), token.str)

          file.out <- data.p(datatype, sprintf("%s_%s", datatype, date.str), ext="gz")

          ## For output
          url.out <- gsub(escapeRegEx(token.str), "", URL)

          if (fileDownloaded(file.out, processed.subfolder=processed.subfolder, minSize=0, check=TRUE)) {
            results="previously downloaded"
          } else {
            downloadBinFile(URL, file.out)
            results <- if (fileDownloaded(file.out, processed.subfolder=processed.subfolder, minSize=0, check=TRUE)) "success" else "ERROR"
          }

          ret_rest.ll[[i]] <- c(date=date.str, datatype=datatype, country=NA_character_, results=results, url=url.out, file=file.out)
       }
    ret_rest <- do.call(rbind, ret_rest.ll)
}



DT.SpotifyAPI_V2_downloadedFiles_info <- as.data.table(rbind(ret_streams, ret_rest))
if (!TEST_RUN)
  jesusForData(DT.SpotifyAPI_V2_downloadedFiles_info)
# ---------------------------

#   
#   curl -dgrant_type=client_credentials -dclient_id=orchard -dclient_secret=4BVPsfKMVu4T6HSQ https://ws.spotify.com/oauth/token
#   
#   ## returned token:  NAoqChIKB29yY2hhcmQSABoAJT9xQlQSFG-MGBvjk9ZqFuBEY2Gk2IBTa3bU
#   
  # wget -O TESTING_streams.gz https://ws.spotify.com/analytics/api/theorchard/streams/2014/10/01/US?oauth_token=NAoqChIKB29yY2hhcmQSABoAJT9xQlQSFG-MGBvjk9ZqFuBEY2Gk2IBTa3bU
  # wget -O streams_US_20141013.gz https://ws.spotify.com/analytics/api/theorchard/streams/2014/10/13/US?oauth_token=NAoqChIKB29yY2hhcmQSABoAJT9xQlQSFG-MGBvjk9ZqFuBEY2Gk2IBTa3bU
#   
#   wget -O tracks.gz https://ws.spotify.com/analytics/api/orchard/tracks/2014/10/01?oauth_token=NAoqChIKB29yY2hhcmQSABoAJT9xQlQSFG-MGBvjk9ZqFuBEY2Gk2IBTa3bU