## TODO:  Look into S3's Enable Access Login feature



startAWStools <- function() {
## I believe I am not using this at all. 
## Instead, I am using the s3_* functions wrapping s3cmd

  AWSACCESSKEY <- getOption("aws_aki")
  AWSSECRETKEY <- getOption("aws_sak")
  lib(AWS.tools, quietly=TRUE)
}


justFor <- function(name, prefix_to_file_name="", root=getOption("aws_default_root"), ...) {
## Creates a full bucket address to the scratch bucket that is specifically for someone else

  s3_p("justForYou", name, prefix=prefix_to_file_name, root=root, ...)
}


s3_bucketExists <- function(bucket, allow_odd_bucket_names) {
  if (!(substr(bucket, 1, 5) == "s3://") && !allow_odd_bucket_names)
    stop ("\nArgument 'bucket' should start with 's3://'\n\nbucket given is:  '", bucket, "'\nHINT: use bucketExists(bucket, allow_odd_bucket_names=TRUE) ")
  cmd <- sprintf("s3cmd ls %s", bucket)
  as.logical(length(system(cmd, intern=TRUE)))
}


s3_p <- function(..., prefix="", root=getOption("aws_default_root"), file="", ext="", add_trailing_underscore_to_prefix=TRUE, ignore_blanks=TRUE, default="DEPRECATED", HINT="prefix s3/bucket/prefix_001.tsv", warn_on_as.path=TRUE) {
## file :: similar to prefix, except that it will have ext appended and never an underscore appended
##
##   vvvvv   NOT SURE IF THIS IS TRUE??
## REMEMBER: prefix will be applied to the filename. It has no impact on the actual path. 
## TODO: Not vectorized on prefix / file

  ## prefix, by defaulting to "" ensures a closing "/"
  if (!missing(default)) {
    warning ("'default=' has been depracated. Use 'root=' instead")
    if (!missing(root))
      stop ("cannot specify both default and root. default has been deprecated, please use root")
    root <- default
  }

  if (missing(root) && is.null(root))
    warning ("'root' bucket value is set to NULL\nHINT: If this is not deliberate, make sure to set\n       options(aws_default_root = <desired root>)")

  ## check if there is already 
  has_root <- FALSE
  if (!missing(...) && (missing(root) || is.null(root))) {
    prev_root <- attr(..1, "root")
    if (!is.null(prev_root)) {
      has_root <- TRUE
      root <- prev_root
    } else if (grepl("^s3:", ..1)) {
      has_root <- TRUE
      root <- ..1  ## the 's3:' will be cleaned later
    }
  }

  ## INPUT ERROR CHECK
  is.char_of_length1(prefix, fail=TRUE, hint="use prefix=\"\" to set to blank")

  ## CHECK THAT root DOES NOT CONTAIN 's3://'
  root <- gsub("^s3:/+", "", root, ignore.case=TRUE)


  if (add_trailing_underscore_to_prefix && nchar(prefix) && !grepl("_$", prefix))
    prefix <- paste0(prefix, "_")

  if (nchar(prefix) & nchar(file))
    stop ("only one of 'prefix' and 'file' may be used")

  ## THREE Things happen here: 1. group dots into a vector (via spaply and list) 2. convert to string 3. remove trailing slash
  dots <- sapply(list(...), function(x) sub("/$", "", as.character(x)))

  if (ignore_blanks) {
    dots <- dots[!sapply(dots, function(x) (!length(x) || !nchar(x)))]
  }


  ## If file is given, use file in place of prefix
  if (nchar(file))
    prefix=file

  args <- c(dots, prefix)
  ## add preface if root is already not present
  if (!has_root)
    args <- c("s3:/", root, args)

  ## test as.path vs paste
  ret.as.path <- do.call(as.path, c(as.list(args), ext=if (!nchar(file)) ext, fsep="/")) %>% gsub("^s3://?", "s3://", .)
  ret.paste <- do.call(paste, c(as.list(args), sep="/"))

  if (ret.paste != ret.as.path  &&  warn_on_as.path) {
    if (gsub("/$", "", ret.paste) == ret.as.path) {
        ## No longer need this warning
        # warning("s3_p produced different results if using paste() than as.path() --  paste() adds a trailing '/'", call.=FALSE)
    } else {
        warning("s3_p produced different results if using paste() than as.path()\n     paste:  ", ret.paste, "\n   as.path:  ", ret.as.path, "", call.=FALSE)
    }
  }

  ret <- ret.paste

  setattr(ret, "root", root)
  setattr(ret, "prefix", prefix)

  return(ret)
}


s3_ls <- function(bucket=getOption("aws_default_root"), recursive=FALSE, check_if_exists=TRUE, flags="", max.nchar=5e4, verbose=TRUE) {
  if (check_if_exists && !s3_bucketExists(bucket)) {
    warning ("bucket '", bucket, "' does not exist -- returning NULL")
    return (NULL)
  }

  if (is.null(flags))
    flags <- ""

  if (isTRUE(recursive))
    flags <- paste(flags, "-r")

  cmd <- sprintf("s3cmd ls %s %s", flags, bucket)

  if (length(cmd) > 1)
    cmd <- pasteC(cmd, C=";")

  nc <- nchar(cmd)
  if (nc > max.nchar)
    stop ("cmd character length (", nc, ") exceeds max.nchar (", max.nchar, ")\nHINT: use eg  max.nchar=1e5  ")

  ret <- system(cmd, intern=TRUE)

  if (verbose)
    cat("\n", ret, "", sep="\n")

  return(invisible(ret))
}


s3_download <- function(bucket, file.local, folder.local=data.p, skip.existing=TRUE, recursive=TRUE, max.nchar=5e4) {
  if (missing(file.local))
    file.local <- gsub(".*/", "", bucket) %>% gsub("_$", "/.", .)
  if (is.function(folder.local))
    folder.local <- {folder.local}()

  dl.location <- as.path(folder.local, file.local)
  if (grepl("/$", dl.location))
    dir.create(dl.location, showWarnings=FALSE)
  if (!file.exists(dirname(dl.location)))
    dir.create(dirname(dl.location))
  cmd <- sprintf("s3cmd get %s %s", bucket, shellClean(dl.location))

  if (skip.existing)
    cmd <- paste(cmd, "--skip-existing")
  if (recursive)
    cmd <- paste(cmd, "--recursive")

  if (length(cmd) > 1)
    cmd <- pasteC(cmd, C=";")

  nc <- nchar(cmd)
  if (nc > max.nchar)
    stop ("cmd character length (", nc, ") exceeds max.nchar (", max.nchar, ")\nHINT: use eg  max.nchar=1e5  ")

  system(cmd, intern=TRUE)
  return(dl.location)
}

## This is a personal function
subbucketNameFromFileName <- function(filename
  # , root_bucket=getOption("aws_default_root")
  , base_folder=if (exists("srcDir")) srcDir else "" # NOTE: src also gets checked for 'ingest', 'out', 'data', etc
  , src_adjust=TRUE
  , verbose=TRUE
  ## NOTE: 'isfolder' was called 'isdir' and called the utilsRS function is.dir();  however, I changed is.dir to isdir() and now introduced error, so changing the name of the argument
  ##        I am not taking the proper steps to deprecate it, because I do not believe it is directly referenced anywhere
  , isfolder = isdir(filename, showWarnings=FALSE)
  ) {
## ARGS
##  isfolder :: logical.  Whether or not filename is a directory. If isfolder is FALSE, basename(filename) will be ignored in the return value.  All values for isfolder other than TRUE (ie, NA, NULL) are interpreted as FALSE. 

  ## If multiple files given, iterate with sapply()
  if (length(filename) > 1) {
    ARGS <- collectArgs(except="filename")
    return(sapply(filename, function(f) do.call(subbucketNameFromFileName, c(filename=f, ARGS))))
  }

  ## if filename is not a folder, drop the basename
  if (!isTRUE(isfolder)) {
    filename <- dirname(filename)
    if (filename == ".")
      filename <- ""
  }

  ## create a path.expand 'd copy
  filename_pe <- path.expand(filename)

  ## if using srcDir variable, check if file is in a compariable dir
  if (isTRUE(src_adjust) && exists("srcDir") && srcDir == base_folder) {
    pat <- gsub("/src/", "/\\\\w{3,11}/", path.expand(srcDir))
    pat <- gsub("/git/", "/git(Data)?/", pat)
    pat <- paste0("^", pat, "/*")
    if (grepl(pat, filename_pe))
      return(gsub(pat, "", filename_pe))
  }

  pat <- path.expand(base_folder)
  pat <- paste0("^", pat, "/*")
  ret <- gsub(pat, "", filename_pe)

  ## if after cleaning, the end result is the same as the path-expanded filename
  ## then use the original (unpath expanded) filename
  if (ret == filename_pe)
    ret <- filename

  ## remove any leading "/"
  ret <- gsub("^/+", "", ret)
  return(ret)
}


if (FALSE) {
  files <- dir("~/git/TEST_DIR", full=TRUE)
  file.local <- c("/home/rsaporta/git/orch/out/Snowflake_Scratch/KinaGrannis.tsv", "/home/rsaporta/git/orch/out/Snowflake_Scratch/KinaGrannis.tsv2")
  s3_upload_and_share(files, "Test_Group")

  s3_upload_and_share(file=.f.out.writeDT, for_whom="retail_marketing")
  s3_upload_and_share(file=files.test, for_whom="TESTING")
  s3_upload_and_share(file.local[[1]], "Ricks Use", rick="rsaporta@gmail.com", 'rick saporta' = "rsaporta@theorchard.com")
  s3_upload_and_share(file.local, "Ricks Use", rick="rsaporta@gmail.com", 'rick saporta' = "rsaporta@theorchard.com", descriptions=c("Stores by Country", "Countries by Store"))

  ## Showing this works
  bucket_to_url('s3://dev-rsaporta/justForYou/retail_marketing/KinaGrannis.tsv')

  ## WORKFLOW
  ## Given some files recently uploaded
  files.s3 <- c("s3://dev-rsaporta/justForYou/TESTING/Vodafone_activity.csv", "s3://dev-rsaporta/justForYou/TESTING/Vodafone_breakage.csv", "s3://dev-rsaporta/justForYou/TESTING/Vodafone_revenue.csv", "s3://dev-rsaporta/justForYou/TESTING/spotify_fact_analytics_count.tsv", "s3://dev-rsaporta/justForYou/TESTING/spotify_raw_row_counts.tsv", "s3://dev-rsaporta/justForYou/TESTING/spotify_raw_row_counts_by_day.tsv")
  ## Sign them
  ret <- s3_sign(files.s3)
  ## Convert to short_url
  shorts <- url_shorten(ret)
}

bucket_to_url <- function(bucket) {
  sub("^s3:/+", "https://s3.amazonaws.com/", bucket)
}

s3_sign <- function(bucket_with_objects, hours=1.5, minutes=0, showWarnings=!(missing(hours) & missing(minutes))) {

  if (any(wh.err <- !grepl("^s3://", bucket_with_objects, ignore.case=TRUE)))
    stop("All bucket addresses must start with s3://..\nOffending bucket(s):\n   ", pasteC(bucket_with_objects[wh.err], C="\n   "))
  total_seconds <- ((hours * 60) + minutes) * 60
  if (total_seconds == 0) {
    warning("attempted to sign S3 bucket with 0-seconds time frame. Defaulting to 60 minutes.")
    total_seconds <- 3600
  }

  expiry_epoch <- as.integer(now() + total_seconds)
  cmd <- bucket_with_objects %>% shellClean %>% sprintf("s3cmd signurl %s %i", ., expiry_epoch) %>% pasteC(C=";")

  # cat("\n---------------------------------\n\n", cmd, "\n\n---------------------------------\n\n")

  signed_urls <- system(cmd, intern=TRUE)
  return(signed_urls)
}

s3_upload_and_share <- function(file.local
  ## bucket Args
  , for_whom
  , bucket = justFor(name=for_whom)
  , hours=24

  ## These are for bitly, and could also be used for the email msg
  , descriptions=NULL

  , skip.existing=TRUE
  , recursive=TRUE
  , max.nchar=5e4
  , verbose=TRUE
  , subject="Your data is available for download"
  , body="DON'T USE"
  , cc=getRS()
  , ...
    ) {

  if (missing(for_whom))
    stop ("'for_whom' needs to be given explicitly. It should be a person, team or department")
  
  ## for_whom dictates the bucket. Clean it up
  for_whom %<>% tolower %>% trim %>% spaceToUnderscore

  ## When the file will be expiring. This is used in the email body
  expiring_on <- now() + hours * 3600
  
  ## upload the files, banking their addresses
  files.s3 <- s3_upload(file.local=file.local, bucket=bucket, subbucket_relative_to_root=NULL, root_bucket=NULL, skip.existing=skip.existing, recursive=recursive, max.nchar=max.nchar, verbose=verbose)

  ## Create signed URLs that are temproarilly public
  signed_urls <- s3_sign(files.s3, hours=hours)

  ## The signed URLs are ridiculously long.  Shorten them with bit.ly
  short_urls <- valueIfErr(url_shorten(signed_urls, USE.NAMES=FALSE), signed_urls)

  ## Format the body of the email. Adding a nice description and bullet-pointing the files
  {
    L <- length(short_urls)
    sep <- "\n"
    if (length(descriptions) == L)
        short_urls %<>% sprintf("<B>%s</B>%s&nbsp;&nbsp;&nbsp;%s", descriptions, sep, .)
    sep <- paste0("\n", sep, "<LI> ")
    body <- sprintf("Your %s available for download from the following %s <UL>%s%s</UL>\n\nFor security, these links are only valid for 24hrs.\n<B>These links expire on %s</B>\nIf you need an updated link please contact Rick Saporta\n\n\n", plrl("files are", L), plrl("URLs", L), sep, pasteC(short_urls, C=sep), timeStamp(time=expiring_on, human=TRUE)) %>% gsub("\\s*<UL>\\s*<LI>", "<UL><LI>", .)
  }

  ## Send the email
  caught <- try({quickEmail(cc=cc, ..., body=body, subject=subject)})
  if (isErr(caught)) {
    warning("quickEmail() failed.  Here is the body of the email\n\n", body, "\n\n")
  }


  return(files.s3)
}


s3_upload <- function(file.local
  ## bucket Args
  , subbucket_relative_to_root=subbucketNameFromFileName(file.local)
  , root_bucket=getOption("aws_default_root")
  , prefix=""
  , add_trailing_underscore_to_prefix=TRUE
  , bucket = s3_p(subbucket_relative_to_root, root=root_bucket, prefix=prefix, add_trailing_underscore_to_prefix=add_trailing_underscore_to_prefix)

  , chunk_size_in_MBs=64 ##  --multipart-chunk-size-mb=SIZE
  , skip.existing=TRUE
  , recursive=TRUE
  , max.nchar=5e4
  , verbose=TRUE
  
) {


  ## ---------------------------------------------------------------------------------
  ## TODO -- We could potentially vectorize over 'file.local' and 'bucket'
  ##         but would have to check for matching lengths etc. 
  ##         For now, disallow multiple buckets
  ## ---------------------------------------------------------------------------------
  ## Multiple files are allowed.  However, they should all be going to the same bucket.
  ## If not, a separate function call to s3_upload() should be used for each file
  bucket <- unique(bucket)
  if (length(bucket) != 1) {
    stop ("bucket must be length 1. ", if (length(file.local) > 1) "If iterating over multiple files, they should all be going to the same place. \nHINT: Use  subbucket_relative_to_root=subbucketNameFromFileName(file.local[[1]])")
  }
  ## ---------------------------------------------------------------------------------


  if (any(!file.exists(file.local))) {
    missing_files <- unname(file.local)[!file.exists(file.local)]
    if (length(missing_files) > 4) 
        missing_files <- c(missing_files[1:3], sprintf(" [+ %i more...]", length(missing_files) - 3 ))
    stop ("file", ifelse(length(missing_files) > 1, "s", ""), "\n   ", pasteC(sprintf("'%s'", missing_files), C=",\n   "), "\ncould not be found")
  }

  cmd <- sprintf("s3cmd put %s %s", pasteC(shellClean(file.local), C=" "), bucket)

  if (skip.existing)
    cmd <- paste(cmd, "--skip-existing")
  if (recursive)
    cmd <- paste(cmd, "--recursive")
  if (!is.null(chunk_size_in_MBs)) {
    if (!is.integer(chunk_size_in_MBs) || is.na(chunk_size_in_MBs))
      warning("chunk_size_in_MBs should be an integer. Will use default (15MB)")
    else
      cmd <- paste(cmd, "--multipart-chunk-size-mb=", chunk_size_in_MBs)
  }

  if (length(cmd) > 1)
    cmd <- pasteC(cmd, C=";")

  nc <- nchar(cmd)
  if (nc > max.nchar)
    stop ("cmd character length (", nc, ") exceeds max.nchar (", max.nchar, ")\nHINT: use eg  max.nchar=1e5  ")

  out_from_cmd <- system(cmd, intern=TRUE)

  ## clean up the default s3cmd output
  if (verbose) {
    msg.out <- sprintf("(File )?'%s/", path.expand("~")) %>% sub("'~/", out_from_cmd) %>% sub("\\s*\\((\\s*\\d+) bytes.*\\).*$", "", .) %>% sub("stored as", "~~>", .)
    bytes <- gsub(".*\\((\\s*\\d+) bytes.*\\).*$", "\\1", out_from_cmd) %>% as.numeric %>% formatBytes
    sprintf("%s (%s)\n", msg.out, bytes) %>% cat(sep="")
  }

  ## Output will be something like
  ##   File '/home/rsaporta/git/orch/out/Snowflake_Scratch/kina_grannis.tsv' stored as 's3://dev-rsaporta/justForYou/retail_marketing/kina_grannis.tsv'
  ## Parse it
  ret <- sub("^.*stored as '(.*)'.*?$", "\\1", out_from_cmd)

  ## confirm via basenames.  (add prefix to file.local)
  {
    ## &&&& TODO 20151201 : Put in a check for when file.local is a folder
    base_local <- paste(prefix, basename(file.local), sep=ifelse(nchar(prefix) && add_trailing_underscore_to_prefix, "_", ""))
    base_ret   <- basename(ret)
    if (any(wh.err <- sort(base_local) != sort(base_ret)))
      warning(if (isdir(file.local)) "----- the below message may be skewed by the fact that we uploaded a folder and not a file -----\n\n", warningCols("Upload may not be correct for file(s)", file.local[order(base_local)][wh.err], cols=1))
  }

  return(ret)
}

# Put file into bucket
#     s3cmd put FILE [FILE...] s3://BUCKET[/PREFIX]


s3cmd <- function(..., verbose=TRUE) {
## generic s3cmd call
  dots <- unlist(list(...))
  cmd <- paste("s3cmd", paste(dots, collapse=" "))
  if (verbose) 
    cat("EXECUTING\n   ", cmd, "\n")
  system(cmd, intern=TRUE)
}

s3_sync <- function(from, to, put_insted_of_sync=FALSE, recursive=all(file.info(from)$is.dir), dry.run=FALSE, skip.existing=FALSE, delete.removed=FALSE, verbose=TRUE) {

  if (length(to) != 1)
    stop ("'to' must be length exactly 1")
  if (!length(from) || !all(file.exists(from)))
    stop ("'from' has no length or does not exist")

  if (any(grepl("/$", from))) 
    warning ("'from' ends in a trailing slash (/)\n\nThe effect is that the folder name will NOT be sent to S3. ie,\n  instead of:  ", as.path(to, basename(from[grepl("/$", from)][[1]]), "file.csv"), "\n  will use:    ", as.path(to, "file.csv"))


  ## Clean up the 'from' argument, including its names
  {
    if (anyDuplicated(from)) {
      warning ("There are duplicates in 'from':\n   ", pasteC(from[duplicated(from)], C="\n   "))
      from <- from[!duplicated(from)]
    }

    if (is.null(names(from)) || any(names(from) == ""))
      names(from) <- basename(from)

    if (anyDuplicated(names(from)))
      names(from) <- from
  }

  flags <- paste0(
            if(isTRUE(recursive))      " --recursive"
          , if(isTRUE(dry.run))        " --dry-run"
          , if(isTRUE(skip.existing))  " --skip-existing"
          , if(isTRUE(delete.removed)) " --delete-removed"
          , ""
          )

  cmds <- sprintf("s3cmd %s%s %s %s"
              , ifelse(isTRUE(put_insted_of_sync), "put", "sync")
              , flags, from, to)

  data.table::setattr(cmds, "names", names(from))

  ret <- emptylist(cmds)
  for (nm in names(cmds)) {
    verboseMsg(verbose, cmds[[nm]], time=FALSE)
    ret[[nm]] <- try(system(cmds[[nm]], intern=TRUE))
  }

  return(ret)
}