# ------------------------------------------------------------------------------------ #




---------------------------------------------------------------------------------------
NOTE TO SELF: 
---------------------------------------------------------------------------------------
Process Part 01 
    This reads all the JSON files, processes them, creates both an RDS (of the data.table)
     and a CSV file for each day
    This file (part 01) only needs to be ran once.  Output is saved to disk
    This is current upto and including  2015-04-30 files
---------------------------------------------------------------------------------------












source("~/git/orch/src/Spotify Playlist Webscrape/00 Featured Playlist Setup.r")

archive_folder <- ???

## The archive folder structure will be 
##  <dataDir> / featured_playlists_raw_json / Archive / DATE / DATETIME / file
## Whereas the original structure is 
##  <dataDir> / featured_playlists_raw_json / DATE / DATETIME / file

data.p(subfolder_for_featuredplaylists, "archive")
##########   READ THE FILES   #############

makeArchiveFilename <- function(filename, levels_back=2) {
???
}

api_info <- "api.spotify.com/v1"

date_folders_for_featuredplaylists <- dir(data.p(subfolder_for_featuredplaylists), recursive=FALSE, include.dirs=TRUE, full.names=TRUE)


## TODO -- this stuff is hardcorded ... needs to be fixed
{
  date_folder <- date_folders_for_featuredplaylists[[1]]
  csv_file <- as.path(dirname(dirname(date_folder)), "featured_playlists_parsed_csv", basename(date_folder), ext="csv")

  ## make this cleaner
  csv_folder <- dirname(csv_file)
  dir.create(csv_folder, showWarnings=FALSE, recursive=TRUE)
}

## CHECK THIS ONE: 
# 2014-06-10 20:00:00,"US","spotify:user:spotify:playlist:1yHZ5C3penaxRdWR7LRIOb","New Music Tuesday","1yHZ5C3penaxRdWR7LRIOb",FALSE,63,"spotify","user","Grab the music to go tonight.",1

## REMINDER:
##   We are downloading the LIST OF PLAYLISTS that were featured at each given date-time-territory combination
##   File hierarchy is
##   DATE -|
##         TIME -|
##               file_by_TERRITORY - pt 01 
##               file_by_TERRITORY - pt 02 [etc]
##         
##         
##   Each file has a LIST of palylists
##   This tells us which playlists were featured that day/time
##   No where here do we have information on WHAT TRACKS or HOW MANY STREAMS that palylist received
##   In order to get that information, we need pull data for each playlist
##   That is a separate process

for (date_folder in date_folders_for_featuredplaylists) {
  ## get the datetime folder inside of date_folder
  folders_for_featuredplaylists <- dir(date_folder, recursive=FALSE, include.dirs=TRUE, full.names=TRUE)
  csv_file <- as.path(dirname(dirname(date_folder)), "featured_playlists_parsed_csv", basename(date_folder), ext="csv")

  ll_ret <- emptylist(folders_for_featuredplaylists)
  for (folder in folders_for_featuredplaylists) {
    files <- extractFilesFromFolder(folder=folder, pattern="^featuredPlaylists", ext="json", full.names=TRUE, recursive=TRUE)
    ret <- lapply(files, function(f) {
    ## TODO:  Break out into standalone function
        J.parsed <- jsonlite::fromJSON(f)
        items <- J.parsed$playlist$items

        playlist_uri   <- items$uri
        playlist_name <- items$name
        playlist_id <- items$id
        playlist_is_collaborative <- items$collaborative
        playlist_total_tracks <- items$tracks$total
        owner_id <- items$owner$id
        owner_type <- items$owner$type

        ## single elements for the whole table
        msg <- J.parsed[["message"]]
        filename <- basename(f)
        country  <- gsub(".*country_(\\w+?)\\+.*$", "\\1", filename)
        folder <- basename(dirname(f))
        timestamp <- as.POSIXct.iso8601(folder)
        total_playlists_featured <- J.parsed$playlists$total

        is_there_more <- !is.null(J.parsed$playlists$`next`)

        cat(as.character(timestamp), " ", filename, "\n")

        if (!grepl("^\\d+T\\d+$", folder))
          warning ("Improper timestamp in folder '", folder, " of file '", filename, "'")

        if (!all(c(
              length(playlist_id) == length(owner_id)
            , length(playlist_id) == length(playlist_name)
            , length(playlist_id) == length(owner_type)
            , length(playlist_id) == length(playlist_uri)
          )))
        warning ("Not all lengths of playlist_id, playlist_name, owner_id, owner_type, playlist_uri  are equal for file '", filename, "'", call.=FALSE)
    
        data.table(
            timestamp
          , country
          , playlist_uri
          , playlist_name
          , playlist_id
          , playlist_is_collaborative
          , playlist_total_tracks
          , owner_id
          , owner_type
          , message=msg
          , total_playlists_featured
          , file = filename
          , api_info = api_info
        )
      })
    ll_ret[[folder]] <- rbindlist(ret)
  }

  ## Flatten
  nm.DT.featured_playlists <- paste0("DT.featured_playlists_", basename(date_folder))
  assign(x=nm.DT.featured_playlists, value=rbindlist(ll_ret), envir=globalenv())

  ## Save RDS and CSV
  jesusForData(objNames=nm.DT.featured_playlists, info=sprintf("datefolder is %s", basename(date_folder)), sub="featuredPlaylists_parsed_DTs", git=FALSE)
  write.table(x=get(nm.DT.featured_playlists, envir=globalenv()), file=csv_file, append=FALSE, sep=",", row.names=FALSE, col.names=TRUE, quote=TRUE, fileEncoding="UTF-8")

  ## Clean up
  rm(list=nm.DT.featured_playlists)
}