setScience("Spotify_API_V2", create=TRUE, subl=FALSE, load=FALSE)
lib(RCurl)
lib(rjson)
lib(doMC)


registerDoMC(24)

TEST_RUN <- FALSE



## Not iterators, used for building URL
base.url <- "https://ws.spotify.com/analytics/api/theorchard"
token <- getNewSpotifyToken()
token.str <- sprintf("?oauth_token=%s", token)


## DATES
dates.using <- seq(Sys.Date()-1, as.Date("2014-08-01"), by="-1 day")

## DATA TYPES
datatypes.using <- c("streams", "tracks", "users")  # "social" ?

## COUNTRY
country_codes.spotify <- getUniqueCountryCodes(refresh=FALSE)
country_codes.other  <- c()  ## TODO 
country_codes.using <- sort(unique(c(country_codes.spotify, country_codes.other)))

if (TEST_RUN) {
  dates.using <- sample(head(dates.using, 25), 2)
  country_codes.using <- c("US", "SE")
}

## Check country has data 
wh.ct_is_live <- sapply(as.path("https://ws.spotify.com/analytics/api/theorchard/streams/2014/10/01/", country_codes.using, paste0("/?oauth_token=", token)), url.exists)
country_codes.NOT_used <- country_codes.using[!wh.ct_is_live]
country_codes.using <- country_codes.using[wh.ct_is_live]

jesusForData(country_codes.spotify, country_codes.using, country_codes.NOT_used)

## CREATE folders, by file type
sapply(data.p(datatypes.using), dir.create, showWarnings=FALSE)

## Create country folders, just in streams
sapply(data.p("streams", country_codes.using), dir.create, showWarnings=FALSE)

if (isExpired(token))
  token <- getNewSpotifyToken()

### --- unique to streams ------- ##

cat("Iterating over ", length(country_codes.using), " countries:\n   ", commaSep(country_codes.using), "\n")
cat("Iterating over ", length(dates.using), " dates:\n   From ", format(min(dates.using), "%Y-%m-%d"), " to ", format(max(dates.using), "%Y-%m-%d"),  "\n")

{
  ret_streams <- 
  foreach(country=country_codes.using, .combine=rbind) %:%  
    foreach(date=dates.using, .combine=rbind) %dopar% 
     {

          datatype <- "streams"

          date.str  <- format(date, "%Y%m%d")

          ## Create URL & file
          URL <- as.path(base.url, datatype, format(date, "%Y/%m/%d"), country, token.str)

          file.out <- data.p(datatype, country, sprintf("%s_%s_%s", datatype, country, date.str), ext="gz")

          ## For output
          url.out <- gsub(escapeRegEx(token.str), "", URL)

          if (fileExistsWithData(file.out)) {
            results="previously downloaded"
          } else {
            downloadBinFile(URL, file.out)
            results <- if (fileExistsWithData(file.out)) "success" else "ERROR"
          }

          c(date=date.str, datatype=datatype, country=country, results=results, url=url.out, file=file.out)
       }
}


{
  ret_rest <- 
    foreach(datatype=setdiff(datatypes.using, "streams"), .combine=rbind) %:%
      foreach(date=dates.using, .combine=rbind) %dopar%
     {
          date.str  <- format(date, "%Y%m%d")

          ## Create URL & file
          URL <- as.path(base.url, datatype, format(date, "%Y/%m/%d"), token.str)

          file.out <- data.p(datatype, sprintf("%s_%s", datatype, date.str), ext="gz")

          ## For output
          url.out <- gsub(escapeRegEx(token.str), "", URL)

          if (fileExistsWithData(file.out)) {
            results="previously downloaded"
          } else {
            downloadBinFile(URL, file.out)
            results <- if (fileExistsWithData(file.out)) "success" else "ERROR"
          }

          c(date=date.str, datatype=datatype, country=NA_character_, results=results, url=url.out, file=file.out)
       }
}

DT.SpotifyAPI_V2_downloadedFiles_info <- as.data.table(rbind(ret_streams, ret_rest))
jesusForData(DT.SpotifyAPI_V2_downloadedFiles_info)
# ---------------------------

#   
#   curl -dgrant_type=client_credentials -dclient_id=orchard -dclient_secret=4BVPsfKMVu4T6HSQ https://ws.spotify.com/oauth/token
#   
#   ## returned token:  NAoqChIKB29yY2hhcmQSABoAJaEhPlQSFEXn9t9ktIkbRMxEh2rPncD11BR0
#   
#   wget -O streams.gz https://ws.spotify.com/analytics/api/orchard/streams/2014/10/01/SE?oauth_token=NAoqChIKB29yY2hhcmQSABoAJaEhPlQSFEXn9t9ktIkbRMxEh2rPncD11BR0
#   
#   wget -O tracks.gz https://ws.spotify.com/analytics/api/orchard/tracks/2014/10/01?oauth_token=NAoqChIKB29yY2hhcmQSABoAJaEhPlQSFEXn9t9ktIkbRMxEh2rPncD11BR0