# minDate <- today() - 4
# !!!!!!!!!!!!!!!!!!!!!!!!!!!

## ------ CRON ----------- ##
## This should run three times a day
# 0 0,7,19 * * * Rscript -e 'source("~/git/orch/src/Spotify Playlist Webscrape/Kick Off Files/Kick Off for ALL PLAYLISTS in our system.r")'

## ------ MANUAL -------- ##
# git pull; Rscript -e 'source("~/git/orch/src/Spotify Playlist Webscrape/Kick Off Files/Kick Off for ALL PLAYLISTS in our system.r")'
###  ----- OR -----
# setGitBranchToSystem(); .g(); source("~/git/orch/src/Spotify Playlist Webscrape/Kick Off Files/Kick Off for ALL PLAYLISTS in our system.r")


## ---------------------------- ##
##    INITIATE                  ##
## ---------------------------- ##



if (FALSE) 
{
  setGitBranchToSystem()
  .g()
}

## Clear screen (for output)
cls(25)


## INITIALIZE
.us()
source("~/git/orch/src/Spotify Playlist Webscrape/00 Featured Playlist Setup.r")

              
## minDatea
assignIfNotExist("minDate", today())

## Notify myself that job is kicked off
try({ email(subj=sprintf("Starting Playlist API Scrape with minDate='%s'", as.character(minDate)), to=getRS("to")) })


snowflake_inuse <- TRUE
sort_playlists <- TRUE
chunk_size <- 2050
NOTIFY <- FALSE

reversing <- FALSE
use_middle <- FALSE

## SNOWFLAKE SETUP
wh <- "CRON_JOBS_LARGE"
dbname <- "prod"
if (snowflake_inuse) {
  sfWarehouseOn(wh=wh, size="L")
  sfWaitForWarehouse(wh=wh)
  setSnowflake(wh=wh, dbname=dbname)
}


check_for_existing_playlists <- TRUE
setPrompt(full=TRUE)

## ---------------------------- ##

{
  warning("REMINDER TO SELF: For TSV's processed before 2015-06-19, three differences: 
      (1) snapshot_id added
      (2) is_explicit, and is_local  (as opposed to 'explicit' and 'islocal')
      (3) tracknumber changed to tracknumber_on_album
      (4) track_number_in_playlist added
      (5) Text forced to UTF-8
      ", call.=FALSE)
}


{

  if (!isTRUE(check_for_existing_playlists))
    stop ("check_for_existing_playlists is not implemented correctly")
  else
    message("minDate is ", humanDate(minDate))

  ## Function to reverse seq in for loop
  FUNC_rev <- if (reversing) rev else identity
  filter_seq <- if (use_middle) 9:14 else TRUE
  
  message("Running ", ifelse(reversing, "Reverse", "Forward"), ifelse(use_middle, " -- Starting in the middle", ""))
  print(FUNC_rev(1:25)[filter_seq])

  ## Capture output to log file
  sinkfile <- newLogFile(name=paste0("Chunked_Webscrape", ifelse(reversing, "_Rev", "_Fwd"), ifelse(use_middle, "_Middle", "")))
  sinkOn(file=sinkfile, zarchive.old.files=TRUE, width=220)

  verboseMsg(verbose, "Getting All Playlists via get_all_playlists()")
  if (!exists("playlists.all")) playlists.all <- get_all_playlists(wh=wh, dbname=dbname, snowflake_inuse=snowflake_inuse)
  if (sort_playlists) playlists.all <- sort(playlists.all)
  verboseMsg(verbose, "There are ", length(playlists.all), " total playlists to process (before paring down)")

  ## Clean up the uri s
  playlists.all <- gsub (":user:#", ":user:%23", playlists.all)
  playlists.all <- gsub (":user:\\[(.*?)\\]", ":user:%5b\\1%5d", playlists.all)
  playlists.all <- gsub ("ä", "%C3%A4", playlists.all)
  playlists.all <- gsub ("ø", "%C3%B8", playlists.all)


  # if (check_for_existing_playlists) {
  #   <change this to the other one - no DT> DT.playlists.existing <- get_all_playlists_ids_downloaded_already(subdir="playlists_raw_json")
  #   L.before <- length(playlists.all)
  #   <no DT> playlists.all <- setdiff(playlists.all, DT.playlists.existing$uri)
  #   verboseMsg(verbose, "Of the", formnumb(L.before, round=FALSE), "playlists.all to process, only ", formnumb(length(playlists.all), round=FALSE), "remain")
  #   if (length(playlists.all) < 6000)
  #     process_list_of_playlists(playlists.all, token=token, file.process_tracking=src.p("processed", dateStamp("playlists_quick_process"), ext="txt"), skip.git=TRUE, process_tracking.check_whole_folder=FALSE)
  #   stop("[DONE]\nThere were less than 6000 Playlists left and thus we shortcircuited the process")
  # }


  ## CHUNK IF UP, so that if this crashes (due to memory issue etc) all work is not lost
  L <- length(playlists.all)
  inds <- seq.int(L)
  group_by_inds <- 1 + ((inds-1) %/% chunk_size)
  noOfGroups <- ceiling(L / chunk_size)
  seqOfGroups <- seq.int(noOfGroups)
  playlists.chunked <- lapply(seqOfGroups, function(g) playlists.all[group_by_inds == g])
  stopifnot(identical(unlist(playlists.chunked), playlists.all))
  jesusForData(playlists.all, playlists.chunked)

  N.iterations <- if (isTRUE(filter_seq)) length(seqOfGroups) else length(filter_seq)
  for (i in FUNC_rev(seqOfGroups)[filter_seq]) {

    verboseMsg(verbose, "Running Garbage Cleanup")
    gc()

    if (NOTIFY)
      notifyAndEmail(sprintf("Part %02i started [%s]", i, ifelse(reversing, "Rev", "Normal")))
    verboseMsg(verbose, sprintf("\n\nStarting iteration %2i of %i  [%s total playlists to process]", i, N.iterations, formnumb(length(playlists.chunked[[i]]))), func='message')

    if (is_expired_spotify_token(token))
      token <- get_spotify_API_token(refresh=TRUE)

    file.process_tracking <- ifelse(reversing, "_rev", "_fwd") %>% dateStamp %>% sprintf("playlists_processed%s.txt", .) %>% src.p("processed", .)

    process_list_of_playlists(playlists.chunked[[i]], token=token, file.process_tracking=file.process_tracking, minDate=minDate, skip.git=TRUE)
    verboseMsg(verbose, "Finished iteration ", i, func="message")


    ## TODO: Upload any files older than XXXX-XX-XX to S3
    "&&&&  LEFT OFF HERE "

  } # // end for-loop

  sinkOff()
}