dlFilesFromFTP <- function(ftp, local.basefolder=data.p("ftp_sync"), folder=NULL, recursive=FALSE, overwrite=FALSE, pattern.including=NULL, pattern.excluding=NULL, userpwd=NULL, maxSize=1.05e8, verbose=TRUE, curl=NULL, verbose.curl=FALSE, ..., max.recursive=25, counter.recursive=1) {
  ## ftp is the base folder in the ftp site, likewise, local.basefolder is the analogous local folder
  ## folder is a subfolder(s) within ftp to pull files from. An analogous folder will be created/used in local.basefolder
  ## note that max.recursive will count EACH iteration of dlFilesFromFTP. So if looking at a tree structure of a folder, the total NODES in the WHOLE tree (not just leaves) must not exceed max.recursive
  ##  .... on second thought, the above may not be true

  stopifnot(require (RCurl))

  ## RECURSIVE CHECK                      ##
  ## ------------------------------------ ##
    if (counter.recursive > max.recursive) {
      stop ("Recursions in dlFilesFromFTP() have exceed the maximum.\nHINT: set 'max.recursive' to a higher value to allow deeper nesting")
    }
  ## ------------------------------------ ##

  browser(expr=inDebugMode(c("curl", "dlFilesFromFTP")), text=sprintf("in dlFilesFromFTP() at the top - iterator: #%0i, ftp: '%s', folder: '%s'", counter.recursive, ftp, commaSep(folder)))

  ## Error Check
  if (length(ftp) > 1)
    stop ("'ftp' should have length == 1. 'folder' is allowed to have length > 1")
  if (length(local.basefolder) > 1)
    stop ("'local.basefolder' should have length == 1. 'folder' is allowed to have length > 1")

  ## Iterate over folders
  if (length(folder) > 1)
    return(lapply(folder, function(fold) dlFilesFromFTP(ftp=ftp, local.basefolder=local.basefolder, folder=fold, recursive=recursive, pattern.including=pattern.including, pattern.excluding=pattern.excluding, userpwd=userpwd, curl=curl, verbose=verbose, verbose.curl=verbose.curl, ...) ))

  if (identical(folder, ""))
    folder <- NULL

  remote.folder <- as.path(ftp, folder)
  local.folder  <- as.path(local.basefolder, folder)
  dir.create(local.folder, showWarnings=FALSE, recursive=TRUE)

  ## include the userpwd in the curl handle, if userpwd given and curl not set
  curl.orig <- curl ## for recursive calls
  if (is.null(curl)) {
    if (is.null(userpwd))
      curl <- getCurlHandle()
    else 
      curl <- getCurlHandle(userpwd=userpwd)
  }

  ## grab all of the remote files
  ## ... do NOT pass curl to getFileNamesFromFTP  ... it does not work for some reason
  files  <- getFileNamesFromFTP(ftp=remote.folder, userpwd=userpwd, ftp.use.epsv=FALSE, maxSize=maxSize, verbose=verbose.curl, ...) #curl=curlH, cainfo=CAINFO)
  length.of.all.files <- length(files)

  ## Keep only matched patterns
  if (!is.null(pattern.including))
    files <- files[grepl(pattern.including, names(files))]
  if (!is.null(pattern.excluding))
    files <- files[!grepl(pattern.excluding, names(files))]

  verboseMsg(verbose, sprintf("Doawnloading %i files (of %i files total) %s\n FROM:  %s    TO:  %s", length(files), length.of.all.files, if(!is.null(pattern.including)) paste0("matching pattern \"", pattern.including,"\"") else "", remote.folder, local.folder), time=FALSE, func="message" )

  ## IF there are files to pull, pull them, and ret will be the local values of where saved to
  if (length(files)) {
    ret <- as.path(local.folder, names(files))
    setattr(ret, "names", names(files))
    fmt <- paste0("%-", max(105, mnchar(files)+2), "s   -- ")
    for (file.nm in names(files)) {
            verboseMsg(verbose, sprintf(fmt, files[file.nm]), time=FALSE, endl=0)
      if (overwrite || !(file.already.exists <- file.exists(ret[file.nm])))
         writeBin(
           # object = getBinaryURL(url=files[file.nm], curl=curlH, cainfo=CAINFO, userpwd=userpwd, ftp.use.epsv=FALSE)
           object = getBinaryURL(url=files[file.nm], userpwd=userpwd, ftp.use.epsv=FALSE, curl=curl, verbose=verbose.curl, ...)
         , con    = ret[file.nm]
        ) 
      ok.txt <- if (file.already.exists) {if (overwrite) "[ OVERWRITTEN ]" else "[ SKIPPED ]"} else "[ OK ]"
      verboseMsg(verbose, ifelse(file.exists(ret[[file.nm]]), yes=ifelse(file.already.exists, "[ALREADY EXISTS]", "[OK]"), no="[ ERROR ]"), time=FALSE)
    }
  } else {
    ## Create empty vector to return
    ret <- setNames(nm=character())
  }

  if (recursive) {
    ret <- setNames(nm=remote.folder, obj=list(ret))
    folders <- getFileNamesFromFTP(remote.folder, userpwd=userpwd, ftp.use.epsv=FALSE, dir.only=TRUE, verbose=verbose.curl, ...)
    browser(expr=inDebugMode(c("curl2", "dlFilesFromFTP")), text=sprintf("in dlFilesFromFTP() at the BOTTOM - iterator: #%0i, ftp: '%s', folder: '%s'", counter.recursive, ftp, commaSep(folder)))

    verboseMsg(verbose, sprintf("Found %i subfolder%s in %s: %s", length(folders), ifelse(length(folders)!=1, "s", ""), remote.folder, commaSep(names(folders))), time=FALSE)
    if (length(folders))
      ret <- c(ret, dlFilesFromFTP(folder=names(folders), ftp=remote.folder, local.basefolder=local.folder, recursive=recursive, pattern.including=pattern.including, pattern.excluding=pattern.excluding, userpwd=userpwd, curl=curl.orig, verbose=verbose, verbose.curl=verbose.curl, ..., max.recursive=max.recursive, counter.recursive=counter.recursive + 1))
  }

  return(ret)
}


getFolderNamesFromFTP <- function(ftp, userpwd=NULL, ftp.use.epsv=FALSE, recursive=FALSE, ..., fail.if.any.not.exist=FALSE, max.recursive=125, counter.recursive=1) {

  if (!length(ftp))
    return(setNames(nm=character()))

  ## overwrite the argument, regardless of the actual value
  dir.only <- TRUE

  ## Collect the args
  args <- ls()
  selfname_(args)
  args <- lapply(args, function(x) get(x))

  ## if multiple ftp folders received, split them
  if (length(ftp) > 1) {
    return(sapply(ftp, function(x) {
      args$counter.recursive <- args$counter.recursive + 1
      cat("iterating for ftp = ", commaSep(ftp), "\n")
      args$ftp <- x
      do.call(getFolderNamesFromFTP, args)
    }))
  }

  ## Execute
  argsNotTo.getFileNamesFromFTP <- c("recursive", "max.recursive", "counter.recursive")
  ret <- do.call(getFileNamesFromFTP, args[names(args) %ni% argsNotTo.getFileNamesFromFTP])

  ## recurse
  if (isTRUE(recursive) && length(ret)) {
    args$ftp <- ret
    args$counter.recursive <- args$counter.recursive + 1
    ret <- c(ret,
      do.call(getFolderNamesFromFTP, args)
    )
  }

  if(is.null(ret))
    return(NULL)
  return(sort(as.vector(unlist(ret, use.names=FALSE))))
}

getFileNamesFromFTP <- function(ftp, userpwd=NULL, ftp.use.epsv=FALSE, ..., fail.if.any.not.exist=FALSE, dir.only=FALSE, maxSize=1.05e8, size=is.finite(maxSize), verbose=FALSE) {
  stopifnot(require (RCurl))

  ## These variables should be TRUE / FALSE
  size     <- isTRUE(size)
  dir.only <- isTRUE(dir.only)

  ftp.is.len1 <- length(ftp) == 1

  browser(expr=inDebugMode(c("curl", "getFileNames", "getFileNamesFromFTP")), text=sprintf("in getFileNamesFromFTP() at the top - dir.only is %s  and  ftp is %s", dir.only, ftp))

  ## use a forloop instead of vectorized ftp, since want it to stop immediately
  if (fail.if.any.not.exist) {
    for (f in ftp)
      if (!url.exists(f, userpwd=userpwd, ...)) 
        stop ("FALSE was returned by \n     url.exists('", f, "')")
  } else {
    ftp <- ftp[sapply(ftp, url.exists, userpwd=userpwd, ftp.use.epsv=ftp.use.epsv, ...)]
    # ftp <- ftp[sapply(ftp, url.exists, userpwd=userpwd)]
  }

  if (!length(ftp)) {
    warning ("ftp sent to getFileNamesFromFTP() is empty, or none of them existed. Returning NULL\nHINT: check curl options such as userpwd, curl and cainfo")
    return(invisible(NULL))
  }

  if (dir.only || size)  {
    fileinfo <- strsplit(getURL(paste0(ftp, "/"), userpwd=userpwd, ftp.use.epsv=ftp.use.epsv, dirlistonly=FALSE, ...), "\n")
  }

  if (size) {
    fsizes <- lapply(fileinfo, function(x) as.numeric(gsub("(.+\\s+){3}\\s*(\\d+)(\\s+)(.+\\s+){3}.+", "\\2", x)))
    if (isTRUE(is.finite(maxSize))) {
      for (i in seq(fileinfo)) {
        inds.sizes <- fsizes[[i]] < maxSize
        if (any(!inds.sizes)) {
          files.excluding <- fileinfo[[i]][!inds.sizes]
          if (verbose)
            cat("excluding the following for being larger than ", formatBytes(maxSize), ":", pasteC(files.excluding, C="\n"), fill=TRUE)
          ## Drop the large values from fileinfo
          fileinfo[[i]] <- fileinfo[[i]][inds.sizes]
        }
      }
    }
  }

  if (dir.only)  {
    inds.areDir <- lapply(fileinfo, grepl, pattern="^d")
    ## Split on the EIGHTH space
    fnames <- lapply(fileinfo, function(x) gsub("(.+\\s+){8}", "", x))
    ## calling the final output 'filenames' for consistancy. Although really they are folders
    filenames   <- mapply(function(f, d) f[d & f %ni% c(".", "..")], fnames, inds.areDir, SIMPLIFY=FALSE)
  } else {
    filenames <- strsplit(getURL(paste0(ftp, "/"), userpwd=userpwd, ftp.use.epsv=ftp.use.epsv, dirlistonly=TRUE, verbose=verbose, ...), "\n")
  }

  ret <- mapply(function(ftp, files) if (length(files)) {setNames(nm=files, obj=paste0(ftp, "/", files))}
              ,  ftp=ftp, files=filenames, SIMPLIFY=FALSE)

  if("curl" %in% names(list(...))  &&   all(sapply(ret, length) == 0) ) {
    warning ("No results are being returned from getFileNamesFromFTP().\nHINT:  This function does not like the 'curl=' argument.  Consdier not using it here")
  }

  if (ftp.is.len1 && is.list(ret))
    ret <- ret[[1]]

  if (length(ret) == 1 && is.null(ret[[1]]))
    ret <- NULL

  return(ret)
}