# -------------------------------------------------------------------------------------------------------------------------  #
  #  -----------------------------------------------------------------------------------------------------------------------   #
  #                                                                                                                            #
  #           File Name              :  CJ_functions.r                                                                         #
  #           Last Updated Funclist  :  16 Nov 2015,  6:11 PM (Monday)                                                         #
  #                                                                                                                            #
  #           Author Name            :  Rick Saporta                                                                           #
  #           Author Email           :  RickSaporta@gmail.com                                                                  #
  #           Author URL             :  www.github.com/rsaporta                                                                #
  #                                                                                                                            #
  #           Packages Called        :  NA                                                                                     #
  #           Packages Used via NS   :  data.table                                                                             #
  #                                                                                                                            #
  #  -----------------------------------------------------------------------------------------------------------------------   #
  #                                                                                                                            #
  #   CJ.expand_DT_by_columns     ( DT, byCols )                                                                               #
  #   allDatesByCols              ( ..., deprecated="DEPRECATED. USE CJ_allDatesByCols() instead" )                            #
  #   CJ_allDatesByCols           ( DT, dateCol="date", byCols=key(DT), universal.range=FALSE, interval="month" )              #
  #   CJ_values_of_columns        ( DT, cols, na.rm=FALSE )                                                                    #
  #   CJ_unique_values_of_columns ( DT, cols, na.rm=c("FALSE", "whole row only", "any") )                                      #
  #   expandGridByRow             ( DT, vec, suffixes=c(".DT", ".vec"), keyToUse=key(DT), preserveList.vec=TRUE )              #
  #                                                                                                                            #
  #                                                                                                                            #
  #                                                     <END FUNCS>                                                            #
  #  -----------------------------------------------------------------------------------------------------------------------   #
  # -------------------------------------------------------------------------------------------------------------------------  #



CJ.expand_DT_by_columns <- function(DT, byCols, allow.cartesian=FALSE) {
  K <- lapply(DT[, byCols, with=FALSE], unique) %>% do.call(CJ, .)

  setkeyIfNot(copy(DT), byCols, verbose=FALSE)[K, allow.cartesian=allow.cartesian]
}

  

allDatesByCols <- function(..., deprecated="DEPRECATED.  USE  CJ_allDatesByCols() instead") {
  warning("allDatesByCols() has been deprecated.  Use CJ_allDatesByCols")
  CJ_allDatesByCols(...)
}

CJ_allDatesByCols <- function(DT, dateCol="date", byCols=key(DT), universal.range=FALSE, interval="month") {
  ## This is a modified CJ type function, useful to ensure no dates are skipped per group.
  ## for each group (as determined by 'setdiff(byCols, dateCol)' ), 
  ##   this function will create a DT used for join-indexing
  ##   The created index will have all dates in seq, useful to ensure that no dates are skipped

  ## To set the order of dateCol relative to byCols, include dateCol in byCols in its expected position.
  ##
  ## universal.range : if FALSE, takes min/max by byCols. if TRUE takes min/max of entire DT
  ## interval :  the distance between dates
  ## 

  message("This function returns a DT to then join into the original DT.  I am not sure if it even works correctly.\nAlso check out CJ.expand_DT_by_columns()")

  if (dateCol %ni% names(DT))
    stop("'dateCol' ('", dateCol, "') not in the names(DT)")

  if (dateCol %ni% byCols)
    message("NOTE for CJ_allDatesByCols():  To set the order of dateCol relative to byCols, include dateCol in byCols in its expected position.")

  ## will be used for key and ordering
  byCols.orig <- copy(byCols)

  byCols <- setdiff(byCols, dateCol)



  if (universal.range) {
    ._minDate <- min(DT[[dateCol]], na.rm=TRUE)
    ._maxDate <- max(DT[[dateCol]], na.rm=TRUE)
    ret <- DT[, setNames(obj=list(seq(._minDate, ._maxDate, by=interval)), nm=dateCol), keyby=byCols]
  } else {
    ret <- DT[, setNames(obj=list(seq(min(get(dateCol), na.rm=TRUE), max(get(dateCol), na.rm=TRUE), by=interval)), nm=dateCol), keyby=byCols]
  }

  if (all(byCols.orig %in% names(ret)))
    setkeyIfNot(ret, byCols.orig, organize=TRUE, verbose=FALSE)

  return(ret)
}



CJ_values_of_columns <- function(DT, cols, na.rm=FALSE) {
## Finds all unique values in the given column then takes CJ of those values
## Useful for filling values, etc

  ## selfname, but shallow
  names(cols) <- cols

  vals <- lapply(cols, function(col) unique(DT[[col]]))

  if (isTRUE(na.rm))
    vals <- lapply(vals, removeNA)

  if (length(vals) != length(cols) || any(sapply(vals, length)==0)) {
    stop ("Could not find unique values of cols or some of them are nothing but NA and na.rm is set to TRUE")
  }

  return(do.call(CJ, vals))
}


CJ_unique_values_of_columns <- function(DT, cols, na.rm=c("FALSE", "whole row only", "any")) {
## Finds all unique ROWS of columns in DT for just those cols

  if (is.null(na.rm))
    na.rm <- FALSE
  if (!is.logical(na.rm))
    na.rm <- match.arg(na.rm)


  ret <- unique(DT[, cols, with=FALSE], by=cols)

  if (na.rm == "whole row only")
    ret <- ret[!(rowSums(is.na(ret)) == ncol(ret))]
  if (na.rm==TRUE)
    ret <- ret[!(rowSums(is.na(ret)) > 0)]

  return(ret)
}


expandGridByRow <- function(DT, vec, suffixes=c(".DT", ".vec"), keyToUse=key(DT), preserveList.vec=TRUE) { 
##  Take a DT and for each row, expands grid with the given vec  
##
# preserveList.vec : if FALSE, we will attempt to coerce each list element into a DT column. If TRUE, we will leave each element as a DT row
# unlistSingle :  if TRUE, if vec is a list of length 1, it is treated as a vector

  # ----------------------------------------------------- #
  # Error Checks
  # ----------------------------------------------------- #
    # DT should be a data.frame or data.table
    if (!(inherits(DT, "data.frame")))
      stop("DT must be a data.table or data.frame")

    if (length(dim(vec)) > 2)
      stop("vec cannot be more than two-dimensional")
  # ----------------------------------------------------- #

  # ----------------------------------------------------- #
  #  Conver to data.tables
  # ----------------------------------------------------- #
    # if vec is not a data.table, convert to one
    if (!is.data.table(vec)) {
      # check if vec is a list (not a data.frame, data.table, etc)
      if (is.list(vec) && is.null(dim(vec))) {
        # if preserveList is set to TRUE, then we want to use `data.table(vec)` 
        #    not `as.data.table(vec)` as the latter "stands up" the list.
        vec <- if (preserveList.vec) data.table(vec) else as.data.table(vec)
      # check if it is an atomic vector
      } else if(is.null(dim(vec))) {
        vec <- data.table(vec)     
      # else, data.frame, etc, use `as.data.table`
      } else {
        vec <- as.data.table(vec)
      }
    }

    # convert to data.table, if not already. (easier for name-dup resolution)
    if (!is.data.table(DT))
      DT <- as.data.table(DT)
  # ----------------------------------------------------- #


  # ----------------------------------------------------- #
  # Ensure no duplicate names
  # ----------------------------------------------------- #

    # find any names present in both DT & vec
    dupNms <- intersect(names(DT), names(vec))

    # if there are any, resolve by appending suffix
    if (length(dupNms)) {
      setnames(DT,  dupNms, paste0(dupNms, suffixes[[1]]))
      setnames(vec, dupNms, paste0(dupNms, suffixes[[2]]))
    }

  # Note: When replicating DT & vec, one should repeat element(or row)-wise, one should repeat table-wise. 
  #       We will have DT be element-wise for two reasons. 
  #         (1) we can levarge key'ing and 
  #         (2) having vec repeat table-wise allows us to simply use cbind, and leverage R's recycling. 
  #             Alternatively, having vec repeat element wise is a lot more invovled, since we would 
  #             have to account for element-wise reps when vec is a vector and then row-wise when vec has dim. 

  # Make reps of DT
  # -------------------
  # reps is either the number of rows or the length of vec
  reps     <- ifelse(is.null(dim(vec)), length(vec), nrow(vec))
  DT.reppd <- data.table::rbindlist(replicate(reps, DT, simplify=FALSE))
  setkey(DT.reppd)

  # Add in reps of vec -  R will recycle vec automatically
  # -------------------
  DT.reppd <- cbind(DT.reppd, vec)

  # set key if `keyToUse` is not null
  if(length(keyToUse)) {
    if (all(keyToUse == "all")) {
      if ("all" %in% names(DT.reppd))
        warning("Problem with keying the expanded DT:\n  Argument `keyToUse` set to 'all', but there is also a column named 'all'.\n  Using all columns.")
      setkey(DT.reppd)
    }
    else 
      setkeyv(DT.reppd, keyToUse)
  }

  return(DT.reppd)
}