## After Downloading II.r
##
## After Downloading I creates DT.playlist_metadata
## (TODO:  Upload to snowflake)
## This one crops it and investigates


"REMINDER TO SELF - After part 'I' calls this file.  Don't ship this code twice"

{
  kCols.pl <- c("uri", "owner.id", "id")

  ## although, we can calculate uri from id and owner.id, we need it often enough that its best keeping it. In fact, key by it
  cols_not_using <- c("playlist.track.album.markets", "playlist.track.markets", "playlist.description", "playlist.api.timestamp", "playlist.followers.href", "playlist.track.id", "playlist.track.uri", "playlist.track.artist.idcsv", "playlist.track.album.id", "playlist.track.album.uri", "playlist.track.addedby.type", "playlist.track.addedby.uri", "playlist.track.discnumber", "playlist.track.previewurl", "playlist.track.api.itemshref", "playlist.track.api.timestamp", "playlist.type", "playlist.track.type", "playlist.track.album.type", "playlist.external_urls.spotify")
  drop_aftwards <- c("playlist.owner.type", "tsv_file")

  DT.pl <- DT.playlist_metadata[, setdiff(names(DT.playlist_metadata), cols_not_using), with=FALSE]

  ## For some reason there are 24 (at last check) files whose fread did not parse
  ##     for now, just drop these rows
  {
    verboseMsg(verbose, "Determining which uris need to be dropped")
    dropping <- DT.pl[playlist.owner.type == ""][playlist.uri == "" & playlist.id == ""]$tsv_file
    if (length(dropping)) {
      uris <- gsub("(.*)/.+?\\.tsv$", "\\1", dropping) %>% convert_spotify_url_to_uri()
      ## Drop now only the unparsed rows but all rows that pertain to those playists. (only about 22 playlists total)
      verboseMsg(verbose, "Dropping", length(dropping), " playlists")
      DT.pl <- DT.pl[!(playlist.owner.type == "" & playlist.uri == "")] [playlist.uri %ni% uris]
    }
  }
  DT.pl[, (drop_aftwards) := NULL]

  ## JUST TO BE SURE
  invisible(suppressWarnings(DT.pl[, c(cols_not_using, drop_aftwards) := NULL]))

  if (all(c("track.is_orchard_dist", "playlist.track.is_orchard_dist") %in% names(DT.pl))) {
    setnames(DT.pl, "track.is_orchard_dist", "track.is_orchard_dist__using__DT.orch_isrc_upc")
  }

  ## Clean up the names -- remove the playlist prefix
  setnames(DT.pl, removeText("^playlist\\.", names(DT.pl)))

  # kCols.pl <- c("owner.id", "id", "track_number_in_playlist")
  setkeyIfNot(DT.pl, kCols.pl, organize=TRUE, verbose=FALSE)

  ## add some more metrics
  DT.pl[, playlist.tracks_per_album := .N, by=list(uri, track.album.name)]


  ## These are the columns keeping
  {
    ids_sampling <- c("0EvnQDQM1TuuHxWJTsrraN", "0FRnL3pL73ObHVteXGdm3A")
    DT.samp <- copy(DT.pl[id %in% ids_sampling])
    ## sometimes the ID might not be found in DT.pl$id ; in which case, take a random sample
    if (!nrow(DT.samp)) {
      DT.samp <- copy(DT.pl[id %in% sample(unique(id), 2)])
      inds <- sample(nrow(DT.samp), 3) %>% sort
    } else {
      inds <- c(
          which(DT.samp[,     track.is_orchard_dist  & id %in% ids_sampling[[1]]])[1:2]
        , which(DT.samp[,     track.is_orchard_dist  & id %in% ids_sampling[[2]]])[1]
        , which(DT.samp[,   (!track.is_orchard_dist) & id %in% ids_sampling[[2]]])[1]
        )
    }

    DT.samp[, name := gsub("\\s\\(.*$", "", name)]

    message("--------- These are an example of the columns keeping in DT.pl ---------")
    DT.samp[inds] %>% t %>% print
    DT.example_of_playlist_track_data <- t(DT.samp[inds]) %>% as.data.table(keep=TRUE)
    writeDT(DT=DT.example_of_playlist_track_data)
    catn("EXAMPLE SAVED TO DISK:  ", out.p("example_of_playlist_track_data", ext="tsv"))
  }

  jesusForData(DT.pl, verbose=FALSE) %>% print
}



#  owner.id                 "1178806467"             "ribesdecoracio"                "125679636"
#  id                       "2ylomZ8HSwWjmYXpFDNydA" "5pBxvE1qYu2sHVIyvUUyAz"        "30P5VNq8TI8FHxQcc2S4jQ"
#  is_collaborative         "FALSE"                  "FALSE"                         "FALSE"
#  name                     "Íslenskt-by ásta"       "Nadales"                       "Walt Disney World Resort"
#  is_public                "TRUE"                   "TRUE"                          "TRUE"
#  followers.total          "284"                    " 51"                           "538"
#  track.isrc               "ISA111202504"           "GBMNA0913140"                  "USWD10423002"
#  track.name               "Leyndarmál"             "Deck the Halls"                "Zip-A-Dee-Doo-Dah"
#  track.datetimeadded      "2014-05-29T21:58:13Z"   NA                              "2013-01-04T21:13:25Z"
#  track.artist.namecsv     "Ásgeir Trausti"         "Frank Sinatra,Bing Crosby"     "James Baskett"
#  track.artist.numberof    "1"                      "2"                             "1"
#  track.album.name         "Dýrð í dauðaþögn"       "The Very Best Christmas Songs" "Disney's Greatest Volume 1"
#  track.album.producttype  "album"                  "album"                         "album"
#  track.is_local           "FALSE"                  "FALSE"                         "FALSE"
#  track.addedby.id         "1178806467"             ""                              "125679636"
#  track.durationms         "217426"                 " 74973"                        "139560"
#  track.is_explicit        "FALSE"                  "FALSE"                         "FALSE"
#  track.popularity         " 0"                     " 0"                            "47"
#  track.tracknumber        " 4"                     "14"                            "17"
#  track_number_in_playlist "  71"                   "1216"                          " 127"
#  total_tracks_in_playlist " 122"                   "2694"                          " 201"
#  upc                      NA                       "5060122332774"                 NA
#  track.is_orchard_dist            NA                       "TRUE"                          NA