#  Rscript -e "source('~rsaporta/git/orch/src/VidServData/00 Check for YT Files.r')"

## FALSE
if (FALSE) {
  setGitBranchToSystem()
  .g()
  source('~rsaporta/git/orch/src/VidServData/00 Check for YT Files.r')
  showSink()
}
## FALSE

## function to write to logfile
logfile_ftp <- "~/git/orch/log/VidServData/ftp_check.txt"
catf <- function(..., append=TRUE) {
  cat(..., file=logfile_ftp, append=append)
}


setScienceIfNot(proj="VidServData", subProj="IngestYTData", load=FALSE)


## Tiemstamp the ftp log file
catf(as.character(now()), "\t")

catheader("Checking for data files in the filedrop", prel=3, endl=1)

source("~/git/orch/src/VidServData/supportFns/check_filedrop_for_YT_data_files_and_move_them_to_ingestdir.r")

are_there_files_to_process <- try(check_filedrop_for_YT_data_files_and_move_them_to_ingestdir())

## Error catch on that last step
if (isErr(are_there_files_to_process)) {
  catf("ERROR -- something went wrong when checking or trying to move the files from the filedrop location\t\n")

  sinkfile <- newLogFile(name="YTDataIngestion")
  sinkOn(sinkfile)
  hr <- "-----------------------------------------------------------------"
  msg <- "THERE WAS AN ERROR RUNNING THE FUNCTION check_filedrop_for_YT_data_files_and_move_them_to_ingestdir()"
  try({msg <- paste0("\n", hr, "\nTHERE WAS AN ERROR RUNNING THE FUNCTION check_filedrop_for_YT_data_files_and_move_them_to_ingestdir()\n\nHere is the original error:\n", hr, attributes(are_there_files_to_process)$condition$message, "\n", hr, call.=FALSE)})
  warning(msg)
  try({email(to=getToGroup("YT"), cc=getTo("DataRequests"), subj="FTP Problem processing YT Data into Looker", body="The &&&& ")})
  sinkOff()
} else if (are_there_files_to_process) {
  catf("processing\t")
  
  ## Set science
  setScience(proj="VidServData", subProj="IngestYTData", create=FALSE, subl=FALSE, load=FALSE)
  setGitBranchToSystem(); .g();

  message("Files Found -- Turning on sink() and Sourcing Ingest File")
  sinkfile <- newLogFile(name="YTDataIngestion")
  sinkOn(sinkfile)

  success <- try(source("~/git/orch/src/VidServData/01 Ingest YouTubeMGMT Data into Snowflake.r"))

  ## If there is an error, send out an email
  if (isErr(success)) {
    msg_error <- "The script at '01 Ingest YouTubeMGMT Data into Snowflake.r' failed"
    try({msg_error %<>% paste0(" with the following error message:\n----------------------------------\n", attributes(success)$condition$message, "\n----------------------------------\n\nThis means that the files were received OK, but that they were unable to be processed due to the error mentioned above.\n")})
    email(to=getToGroup("YT"), cc=getTo("DataRequests"), subj="Problem processing YT Data into Looker", body=msg_error)
    catf("ERROR -- files were moved successfully, but processing failed.\t\n")
  } else {
    catf("done\n")
    email(to=getToGroup("YT"), cc=getTo("DataRequests"), subj="This is only a test Email -- please ignore", body="This is an example of an email that will go out if the YouTube data failed to load into Looker")


    ## Keep the logfile at a reasonable size. 
    ## Everytime it passes 5,000 lines, drop all but the last 800.
    try({
      if (fRowcount(logfile_ftp) > 5000) {
        readLines(logfile_ftp) %>% tail(800) %>% catf(append=FALSE)
      }
    })

  }


  ## Turn sink off
  sinkOff()

## Otherwise, nothing to do except log it
} else {

  catf("No files to process\t\n")
}