##  These functions check against the files on disk to see which playlists were already downloaded

# There are two locations to check 
# 
# dataDir / playlists_raw_json
#    -- I believe these are just for "featured playlists" (ie, my older workd)
# outDir / scraped_files
#


## ------------------------------------------------------ ##
if (FALSE) {
  filenames <- "playlist_00CmzvhQuDJY4kyuczsmwI+user_pablonzx+dld_20150429+part_02_of_02.json"
  json_filename_to_playlists_uri(filenames)
  get_all_playlists_ids_downloaded_already()
  get_all_playlists_ids_downloaded_already(return="ids")
}
json_filename_to_playlists_uri <- function(filenames) {
  pat.confirm <- "^playlist_(.{22})\\+user_(.+?)\\+dld_\\d{8}\\+part_\\d+_of_\\d+\\.json$"
  if (any(!grepl(pat.confirm, filenames)))
    stop("Not all filenames match the expected filename pattern for a playlist json")

  playlist_ids <- gsub(pat.confirm, "\\1", filenames)
  user_ids <- gsub(pat.confirm, "\\2", filenames)

  uris <- sprintf("spotify:user:%s:playlist:%s", user_ids, playlist_ids)
  data.table(uri=uris, user_id=user_ids, playlist_id=playlist_ids, funky_char=grepl("_", user_ids))
}

if (FALSE) {
  finfo <- file.info(data.p(subdir, filenames)) %>% setDT
  finfo[, filenames := filenames]
  desc(finfo)
  finfo[ctime >= as.POSIXct(today())]

}
get_all_playlists_ids_downloaded_already <- function(subdir="playlists_raw_json", return=c("DT", "ids", "uris"), verbose=TRUE) {
  "playlist_00CmzvhQuDJY4kyuczsmwI+user_pablonzx+dld_20150429+part_02_of_02.json"

  return <- match.arg(return)
  filenames <- dir(data.p(subdir), pattern="\\.json$")

  pat.confirm <- "^playlist_.{22}\\+user_.+?\\+dld_\\d{8}\\+part_\\d+_of_\\d+\\.json$"
  matched <- grepl(pat.confirm, filenames)

  if (!percTrue(matched) == 1)
    warning (sprintf("There are %i out of %i json files in [dataDir]/%s/ that did not match playlist pattern", sum(!matched), length(filenames), subdir))
  else
    matched <- TRUE

  DT.uris_etc <- json_filename_to_playlists_uri(filenames[matched]) %>% unique(by=NULL)
  verboseMsg(verbose, formnumb(nrow(DT.uris_etc), round=FALSE), " playlists found in [dataDir]/", subdir, "/", sep="")

  if (return == "ids")
    return(DT.uris_etc$playlist_ids)
  if (return == "uris")
    return(DT.uris_etc$uri)
  return(DT.uris_etc)
}

## Check the user_ids with funky symbols
if (FALSE) 
{
  DT.uris_etc <- get_all_playlists_ids_downloaded_already()
  if (!exists("playlists.all")) playlists.all <- get_all_playlists()
  matches <- sapply(DT.uris_etc[(funky_char), playlist_id], extract, playlists.all)
  matches[sapply(matches, length) == 0] <- NA_character_
  matches <- unlist(matches)
  DT.uris_etc[(funky_char), matched_uri := matches]
  DT.uris_etc[!(matched_uri == uri)]

  ## -------------------------------

  playlists_in_outDir <- get_all_playlists_ids_downloaded_to_outDir()

}


## EXAMPLES
if (FALSE) {
  get_all_playlists_downloaded_to_outDir(subdir="scraped", return="file") -> Z; cat(length(Z), head(Z, 3), sep="\n")
  get_all_playlists_downloaded_to_outDir(subdir="scraped", return="uri")  -> Z; cat(length(Z), head(Z, 3), sep="\n")
  cat("    ------- \n")
  get_all_playlists_downloaded_to_outDir(subdir="parsed",  return="file") -> Z; cat(length(Z), head(Z, 3), sep="\n")
  get_all_playlists_downloaded_to_outDir(subdir="parsed",  return="uri")  -> Z; cat(length(Z), head(Z, 3), sep="\n")
}

get_all_playlists_downloaded_to_outDir <- function(subdir=c("parsed_json", "scraped_files"), minDate=today() - 7, returnWhat=c("uri", "filename"), removeDuplicates=TRUE, full=FALSE, showWarnings=TRUE, verbose=TRUE) {
## removeDuplicates :: only removes duplicate downloads of the same URL, NOT of the same URI
## ie, if a playlist required paginatin, all of the pages will be returned for that playlist
## if returnWhat is "uri" the removeDuplicates is always true, no matter what

  subdir <- match.arg(subdir)
  returnWhat <- match.arg(returnWhat)

  if (returnWhat == "uri") {
    if (!removeDuplicates)
      warning ("removeDuplicates is ignored when returnWhat = 'uri'")
    if (isTRUE(full))
      warning ("full is ignored when returnWhat = 'uri'")
  }


  ## pattern pieces to match against filenames
  fmt <- "%s\\d{8}_\\d{4,6}_Z\\.%s$"
  fname.base <- if (subdir == "parsed_json") "playlistid-.+" else ""
  ext <- ifelse(subdir == "scraped_files", "json", "tsv")
  pat.files <- sprintf(fmt, fname.base, ext)

  verboseMsg(verbose, "About to read files from '", out.p(subdir), "'  ----   this may take a few minutes", sep="", minw=125)
  filenames <- subdir %>% out.p %>% dir(full=FALSE, recursive=TRUE, pattern=pat.files) %>%
                grep("/users/.+/playlists/", ., value=TRUE) ## Extract just 'playlist' files

  L.total <- length(filenames)
  verboseMsg(verbose, "Extracted ", formnumb(L.total, round=FALSE), " files", sep="", minw=125)


  ## Ensure greater than minDate
  if (!is.null(minDate)) {
      minDate_string <- to_ts_string(minDate)
      verboseMsg(verbose, "Cropping list of files to those downlaoded on or after '", minDate_string, "'", sep="", minw=125)
      if (subdir == "scraped_files") {
        filenames %<>% {.[basename(.) >= minDate_string]}
      } else if (subdir == "parsed_json") {
        filenames %<>% {.[removeText("^playlistid-[A-Za-z0-9]{22,}_", basename(.)) >= minDate_string]}
      } else {
        warning ("Do not know how to check the date on files in subdir '", subdir, "'")
      }
  }

  ## Remove duplicates
  if (removeDuplicates)
    filenames %<>% sort(decreasing=TRUE) %>% {.[!duplicated(dirname(.))]}

  browser(expr=inDebugMode("get_all_playlists_downloaded_to_outDir"), text = "get_all_playlists_downloaded_to_outDir")

  if (!length(filenames)) {
    verboseMsg(showWarnings, "No previously-processed playlists files found in ", out.p(subdir), " with timeStamp greater than ", minDate_string, " -- returning character(0L)", call.=FALSE)
    return(character(0L))
  }

  ret <- filenames
  if (returnWhat == "uri") {
    ret %<>% gsub("(.*/).*", "\\1", .) %>% ## remove everything after the last slash, ie the filename
             gsub("tracks/offset=\\d+&limit=\\d+/$", "", .) %>% ## remove any tracks pagination trailing portion
             unique %>% ## after track portion is removed, there will be many duplicates; make unique
             convert_spotify_url_to_uri ## convert to uri
  } else if (isTRUE(full)) {
    ret %<>% out.p(subdir, .)
  }

  verboseMsg(verbose, sprintf("%s playlist %ss found in [outDir]/%s/  (out of %s total playlist '.%s' files)", formnumb(length(ret), round=FALSE), substr(returnWhat, 1, 4), subdir, formnumb(L.total, round=FALSE), ext))

  return(ret)

}