## ------------------------ ##
##         START HERE       ## 
## ------------------------ ##
setScience("Spotify Playlist Webscrape", create=TRUE, subl=FALSE, load=FALSE)
options(warn=0)
options(verbose_minw = 99)

verbose <- TRUE

subfolder_for_playlists <- "playlists_raw_json"

## for each uri, write the output to file
token <- get_spotify_API_token()

dir.create(data.p(subfolder_for_playlists), showWarnings=FALSE, recursive=TRUE)

if (!exists("DT.uris"))
  get_DT.uris()
uris <- DT.uris$source_uri


## Check against which files were downloaded already
filesDownloaded <- dir(data.p(subfolder_for_playlists), pattern="^playlist.*json$")

DT.uris_parsed  <- parse_spotify_uri(uris)
DT.files_parsed <- parsePlaylistFilename(filesDownloaded, verbose=FALSE)[part == 1]

setkeyIfNot(DT.uris_parsed,  playlist, user, verbose=FALSE, organize=TRUE)
setkeyIfNot(DT.files_parsed, playlist, user, verbose=FALSE, organize=TRUE)


## Grab the URIs that do not have any files
uris_downloaded_already <- suppressWarnings(merge(DT.uris_parsed, DT.files_parsed, all=TRUE, by=c("playlist", "user")) [!is.na(filename), uri])
verboseMsg(verbose, sprintf("%s uris remain to downlaod out of %s total", formnumb(length(setdiff(uris, uris_downloaded_already))), formnumb(length(uris), round=FALSE)))

for (uri in setdiff(uris, uris_downloaded_already)) { 
  try({
    ## PARSE
    parsed <- parse_spotify_uri(uri)
    if (!all(c("user", "playlist") %in% names(parsed))) {
      warning ("uri '", uri, "' failed to parse properly")
      next
    }
    user_id  <- parsed[["user"]]
    playlist <- parsed[["playlist"]]
    verboseMsg(verbose, sprintf("Getting playlist data for user = %13s  &  playlist = %s", user_id, playlist), seconds=TRUE)

    ## refresh token if needed
    if (token$time_end < now() + 300)
      token <- get_spotify_API_token()
    ## Pull raw jsons
    jsons <- get_url_for_spotify_playlist_data(user_id=user_id, playlist=playlist, token=token, verbose=FALSE)
    if (!length(jsons))
      warning ("No JSON returned for uri = '", uri, "'", call.=FALSE)
    file_nms <- sprintf("playlist_%s+user_%s+dld_%s+part_%02i_of_%02i", playlist, user_id, dateStamp(), seq_along(jsons), length(jsons))
    files_out <- data.p(subfolder_for_playlists, file_nms, ext=".json")
    mapply(writeLinesUTF8, text=jsons, con=files_out)
  })
}